From c034e6cf487874d59e523ee84a8fef6c1cf82b6b Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 27 Mar 2024 18:43:36 -0700 Subject: [PATCH 001/136] Seed repo --- .devops.sh | 37 +++++++++ .gitignore | 65 ++++++++++++++- .gitlab-ci.yml | 178 ++++++++++++++++++++++++++++++++++++++++ .golangci.yml | 95 +++++++++++++++++++++ Dockerfile | 34 ++------ Makefile | 55 +++++++++++++ docker-user | 1 + scripts/build.sh | 40 +++++++++ scripts/docker-login.sh | 24 ++++++ scripts/docker.sh | 86 +++++++++++++++++++ scripts/release-tag.sh | 18 ++++ scripts/test.sh | 16 ++++ version | 1 + 13 files changed, 624 insertions(+), 26 deletions(-) create mode 100755 .devops.sh create mode 100644 .gitlab-ci.yml create mode 100644 .golangci.yml create mode 100644 Makefile create mode 100644 docker-user create mode 100755 scripts/build.sh create mode 100755 scripts/docker-login.sh create mode 100755 scripts/docker.sh create mode 100755 scripts/release-tag.sh create mode 100755 scripts/test.sh create mode 100644 version diff --git a/.devops.sh b/.devops.sh new file mode 100755 index 0000000..71f765a --- /dev/null +++ b/.devops.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +DEVOPS_UTILS_URL=${DEVOPS_UTILS_URL:-"gitlab.com/f5/nginx/tools/devops-utils.git"} +DEVOPS_UTILS_REF=${DEVOPS_UTILS_REF:-"master"} +rootdir=$(git rev-parse --show-toplevel) +devops_utils_dir="${rootdir}/.devops-utils" +git_update_file=${devops_utils_dir}/.last-git-update +ttl_seconds=$((60 * 60)) + +if [ "${CI}" == "true" ]; then + url="https://gitlab-ci-token:${CI_JOB_TOKEN}@${DEVOPS_UTILS_URL}" +else + # - change the first occurrence of "/" to ":" for local git clone + url="git@${DEVOPS_UTILS_URL/\//:}" +fi + +epoch=$(date +%s) + +# - get a local copy of devops-utils and update it when it's more than 1h old +if [ ! -d "$devops_utils_dir" ]; then + if ! git clone -q "${url}" "${devops_utils_dir}" --branch "${DEVOPS_UTILS_REF}" --depth 1; then + echo "ERROR: failed to clone devops-utils repo!" + exit 1 + fi + echo "$epoch" > "$git_update_file" +else + if [ $((epoch - $(cat "$git_update_file"))) -gt $ttl_seconds ]; then + cd "$devops_utils_dir" || exit + git fetch -q origin + git reset -q --hard origin/"${DEVOPS_UTILS_REF}" + echo "$epoch" > "$git_update_file" + cd .. + fi +fi + +# shellcheck disable=SC1090,SC1091 +source "${devops_utils_dir}/devops-core-services.sh" diff --git a/.gitignore b/.gitignore index cb5c33a..4e1c0fd 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,67 @@ cover* tmp/ docs/tls/DESIGN.md :q -qqq \ No newline at end of file +qqq.env +.env* +!.env.example +!.allowed_clients.json +!.env.example.auth +*.db +priv/certs +priv/nginx-agent/* +!priv/nginx-agent/nginx-agent.conf.example +key-data.json +nginx-instance-manager.tar.gz +vendor/ + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Output from debugger +__debug_bin + +code-quality.json +coverage/* + +# vim +*~ +*.swp + +### VisualStudioCode (from https://gitignore.io/api/VisualStudioCode) ### +.vscode/* +!.vscode/tasks.example.json +!.vscode/launch.example.json +!.vscode/extensions.json +!.vscode/KubernetesLocalProcessConfig*.yaml +*.code-workspace + +### Goland +.idea/* + +# bridge to kubernetes artifact +/KubernetesLocalProcessConfig.yaml + + +# output directory for build artifacts +build + +# output directory for test artifacts (eg. coverage report, junit xml) +results + +# devops-utils repo +.devops-utils/ + +# Ignore golang cache in CI +.go/pkg/mod + +.go-build diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..08dfa9a --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,178 @@ +--- +include: + - project: "f5/nginx/tools/devops-utils" + file: "/include/cxsast-ci-generic.yml" + ref: "master" + - project: "f5/nginx/tools/devops-utils" + file: "/include/devops-docker-cicd.yaml" + ref: "master" + - project: "f5/nginx/tools/devops-utils" + file: "include/devops-whitesource.yaml" + ref: "master" + - project: "f5/nginx/tools/devops-utils" + file: "/include/gitlab-sast.yml" + ref: "master" + +stages: + - lint+test+build + - release + +variables: + DEVTOOLS_IMG: ${DEVOPS_DOCKER_URL_DEFAULT}/nginx-azure-lb/nlb-devtools:latest + +workflow: + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_BRANCH + +.import-devops-core-services: &import-devops-core-services | + source ${CI_PROJECT_DIR}/.devops.sh + +.go-cache: + variables: + GOPATH: $CI_PROJECT_DIR/.go + cache: + key: + files: + - go.mod + - go.sum + paths: + - .go/pkg/mod/ + +.go-cache-readonly: + extends: + - .go-cache + cache: + policy: pull + +.golang-private: &golang-private + - | + cat << EOF > ~/.netrc + machine gitlab.com + login gitlab-ci-token + password ${CI_JOB_TOKEN} + EOF + go env -w GOPRIVATE="gitlab.com/f5" + go env + +lint + unit-test + build: + stage: lint+test+build + image: $DEVTOOLS_IMG + extends: + - .devops-docker-cicd-large + - .go-cache + script: + - *golang-private + - | + if [ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]; then + time make lint + git diff --exit-code + time make test + fi + time make publish + coverage: '/^total:\s+\(statements\)\s+(\d+\.\d+\%)$/' + artifacts: + when: + always + paths: + - results + expire_in: 3 hours + reports: + junit: results/report.xml + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: '$CI_COMMIT_BRANCH || $CI_MERGE_REQUEST_ID' + +unit-test-data-race: + stage: lint+test+build + image: $DEVTOOLS_IMG + variables: + GO_DATA_RACE: "true" + extends: + - .default-runner-large + - .go-cache-readonly + script: + - *golang-private + - time make test + coverage: '/^total:\s+\(statements\)\s+(\d+\.\d+\%)$/' + artifacts: + when: + always + paths: + - results + expire_in: 3 hours + reports: + junit: results/report.xml + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $GO_DATA_RACE == "true"' + +whitesource-scan: + stage: lint+test+build + extends: + - .default-runner + - .go-cache + - .whitesource-template-go + variables: + NESTED: "true" + WS_WSS_URL: "https://f5.whitesourcesoftware.com/agent" + WS_APIKEY: "${WS_APIKEY_NGINX}" + WS_PRODUCTNAME: "N4A" + WS_PROJECTNAME: "${CI_PROJECT_NAME}" + WS_GO_MODULES_RESOLVEDEPENDENCIES: "true" + WS_GO_RESOLVEDEPENDENCIES: "false" + WS_GENERATEPROJECTDETAILSJSON: "true" + script: + - *golang-private + - !reference [.whitesource-template-go, script] + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + artifacts: + when: always + paths: + - ${CI_PROJECT_DIR}/whitesource/ + expire_in: 1 weeks + +################################################################################ +# GitLab SAST security scanning +# CISO requires that all products be scanned statically for vulnerabilities. +# For more information, please see: +# https://gitlab.com/f5/nginx/tools/devops-utils/-/blob/master/include/gitlab-sast.md +# NOTE: please do not alter, change, modify, or overwrite job rules, job +# variables, or runtime states in order to maintain compliance to CISO +# requirements. +################################################################################ +sast: + stage: lint+test+build + +# ===================================================================================== +# CISO-required, SAST-scanning jobs involving the widely distributed checkmarx vendor +# Job starts immediately and is independent of all other jobs present in the pipeline. +# ===================================================================================== +checkmarx-scan: + stage: lint+test+build + extends: + # Please DO NOT overwrite extended job's image, gitlab-runner tags, or job rules + - .checkmarx-scan-security + variables: + # project specific variables + CX_SOURCES: "." + CX_FLOW_ZIP_EXCLUDE: "" + needs: [] + allow_failure: true + +tag: + stage: release + image: $DEVTOOLS_IMG + extends: + - .default-runner + script: + - ./scripts/release-tag.sh + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..d89f3f5 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,95 @@ +# GolangCI-Lint settings + +# Disable all linters and enable the required ones +linters: + disable-all: true + + # Supported linters: https://golangci-lint.run/usage/linters/ + enable: + - errcheck + - exhaustruct + - gosimple + - govet + - ineffassign + - staticcheck + - typecheck + - unused + - bodyclose + - dupl + - gochecknoinits + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - gosec + - lll + - misspell + - nakedret + - prealloc + - exportloopref + - stylecheck + - unconvert + - unparam + - paralleltest + - forbidigo + fast: false + +# Run options +run: + # 10 minute timeout for analysis + timeout: 10m + skip-dirs-use-default: true + skip-dirs: + - .go/pkg/mod + - pkg/spec/api # Generated code + - vendor + - vendor-fork +# Specific linter settings +linters-settings: + gocyclo: + # Minimal code complexity to report + min-complexity: 16 + govet: + # Report shadowed variables + check-shadowing: true + misspell: + # Correct spellings using locale preferences for US + locale: US + goimports: + # Put imports beginning with prefix after 3rd-party packages + local-prefixes: gitswarm.f5net.com/indigo,gitlab.com/f5 + exhaustruct: + # List of regular expressions to match struct packages and names. + # If this list is empty, all structs are tested. + # Default: [] + include: + - 'gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/pkg/token.TokenID' + - 'gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/internal/dpo/agent/certificates.CertGetRequest' + +issues: + # Exclude configuration + exclude-rules: + # Exclude gochecknoinits and gosec from running on tests files + - path: _test\.go + linters: + - gochecknoinits + - gosec + - path: test/* + linters: + - gochecknoinits + - gosec + # Exclude lll issues for long lines with go:generate + - linters: + - lll + source: "^//go:generate " + # Exclude false positive paralleltest error : Range statement for test case does not use range value in test Run + - linters: + - paralleltest + text: "does not use range value in test Run" + + # Disable maximum issues count per one linter + max-issues-per-linter: 0 + + # Disable maximum count of issues with the same text + max-same-issues: 0 diff --git a/Dockerfile b/Dockerfile index 9aa6b8c..0a12779 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,11 @@ -# Copyright 2023 f5 Inc. All rights reserved. -# Use of this source code is governed by the Apache -# license that can be found in the LICENSE file. +FROM alpine:3.14.1 AS base-certs +RUN apk update && apk add --no-cache ca-certificates -FROM golang:1.19.5-alpine3.16 AS builder +FROM scratch AS base +COPY docker-user /etc/passwd +USER 101 +COPY --from=base-certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -WORKDIR /app - -COPY go.mod go.sum ./ - -RUN go mod download - -COPY . . - -RUN go build -o nginx-loadbalancer-kubernetes ./cmd/nginx-loadbalancer-kubernetes/main.go - -FROM alpine:3.16 - -WORKDIR /opt/nginx-loadbalancer-kubernetes - -RUN adduser -u 11115 -D -H nlk - -USER nlk - -COPY --from=builder /app/nginx-loadbalancer-kubernetes . - -ENTRYPOINT ["/opt/nginx-loadbalancer-kubernetes/nginx-loadbalancer-kubernetes"] +FROM base as nlk +ENTRYPOINT ["/nlk"] +COPY build/nlk / diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4d404ac --- /dev/null +++ b/Makefile @@ -0,0 +1,55 @@ +# - init general vars +BUILD_DIR = build +export BUILD_DIR +RESULTS_DIR = results +export RESULTS_DIR +VERSION = $(shell bash -c 'source version; echo $$VERSION') +export VERSION + +DOCKER_REGISTRY ?= local +DOCKER_TAG ?= latest + +# - init go vars +GOPRIVATE = *.f5net.com,gitlab.com/f5 +export GOPRIVATE + +.PHONY: default tools deps fmt lint test build build.docker publish + +default: build + +tools: + @go install gotest.tools/gotestsum@latest + @go install golang.org/x/tools/cmd/goimports@latest + @go install github.com/jstemmer/go-junit-report@v1.0.0 + @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.57.1 + +deps: + @go mod download + @go mod tidy + @go mod verify + +fmt: + @find . -type f -name "*.go" -exec goimports -e -w {} \+ + +lint: + @find . -type f -name "*.go" -exec goimports -e -w {} \+ + @golangci-lint run -v ./... + +test: + @./scripts/test.sh + +build: + @./scripts/build.sh + +build-linux: + @./scripts/build.sh linux + +build-linux-docker: + @./scripts/docker.sh build + +publish: build-linux build-linux-docker + @scripts/docker-login.sh + @./scripts/docker.sh publish + +clean: + rm -rf $(BUILD_DIR)/ diff --git a/docker-user b/docker-user new file mode 100644 index 0000000..65be48a --- /dev/null +++ b/docker-user @@ -0,0 +1 @@ +nginx:x:101:101:nginx:/var/cache/nginx:/sbin/nologin \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..aba9ad5 --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -ex + +os="$1" + +GIT_DESCRIBE=$(git describe --long --always --dirty) +if [[ -z $CI_COMMIT_SHORT_SHA ]]; then + CI_COMMIT_SHORT_SHA=$(git rev-parse --short=8 HEAD) +fi +if [[ -z $VERSION ]]; then + VERSION=$(source version;echo "$VERSION") +fi + +if [ "$os" == "linux" ]; then + export GOOS=linux + export GOARCH=amd64 + export CGO_ENABLED=0 +fi + +mkdir -p "$BUILD_DIR" + +pkg_path="./cmd/nginx-loadbalancer-kubernetes" +BUILDPKG="gitlab.com/f5/nginx/nginxazurelb/nlk/pkg/buildinfo" + +ldflags=( + # Set the value of the string variable in importpath named name to value. + -X "'$BUILDPKG.semVer=$VERSION'" + -X "'$BUILDPKG.shortHash=$CI_COMMIT_SHORT_SHA'" + -X "'$BUILDPKG.gitDescribe=$GIT_DESCRIBE'" + -s # Omit the symbol table and debug information. + -w # Omit the DWARF symbol table. + -extldflags "'-fno-PIC'" +) + +go build \ + -v -tags "release osusergo" \ + -ldflags "${ldflags[*]}" \ + -o "${BUILD_DIR}/nlk" \ + "$pkg_path" diff --git a/scripts/docker-login.sh b/scripts/docker-login.sh new file mode 100755 index 0000000..1ae5d2f --- /dev/null +++ b/scripts/docker-login.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eo pipefail + +rootdir=$(git rev-parse --show-toplevel) +docker_login_file=${rootdir}/.devops-utils/.last-docker-login + +# - perform a new docker login if last login was more than 1h ago +ttl_seconds=$((60 * 60)) + +epoch=$(date +%s) + +if [ -e "$docker_login_file" ] && [ $((epoch - $(cat "$docker_login_file"))) -lt $ttl_seconds ]; then + exit 0 +fi + +# shellcheck disable=1090 +source "${rootdir}/.devops.sh" +devops.docker.login > /dev/null +if [ "$CI" != "true" ]; then + devops.backend.docker.set "azure.container-registry-dev" + devops.docker.login > /dev/null +fi +echo "$epoch" > "$docker_login_file" diff --git a/scripts/docker.sh b/scripts/docker.sh new file mode 100755 index 0000000..0015f49 --- /dev/null +++ b/scripts/docker.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +set -eo pipefail + +ROOT_DIR=$(git rev-parse --show-toplevel) + +build() { + echo "building image: $image" + DOCKER_BUILDKIT=1 docker build --target "$image" \ + --label VERSION="$version" \ + --label COMMIT="${CI_COMMIT_SHORT_SHA}" \ + --label PROJECT_NAME="${CI_PROJECT_NAME}" \ + --tag "${repo}:${CI_COMMIT_REF_SLUG}" \ + --tag "${repo}:${CI_COMMIT_REF_SLUG}-$version" \ + --tag "${repo}:${CI_COMMIT_SHORT_SHA}" \ + -f "${ROOT_DIR}/Dockerfile" . +} + +publish() { + docker push "$repo:${CI_COMMIT_REF_SLUG}" + docker push "$repo:${CI_COMMIT_REF_SLUG}-$version" + docker push "$repo:${CI_COMMIT_SHORT_SHA}" + if [[ "$CI_COMMIT_REF_SLUG" == "${CI_DEFAULT_BRANCH}" ]]; then + docker tag "$repo:${CI_COMMIT_SHORT_SHA}" "$repo:latest" + docker tag "$repo:${CI_COMMIT_SHORT_SHA}" "$repo:$version" + docker push "$repo:latest" + docker push "$repo:$version" + fi +} + +init_ci_vars() { + if [ -z "$CI_COMMIT_SHORT_SHA" ]; then + CI_COMMIT_SHORT_SHA=$(git rev-parse --short=8 HEAD) + fi + if [ -z "$CI_PROJECT_NAME" ]; then + CI_PROJECT_NAME=$(basename "$ROOT_DIR") + fi + if [ -z "$CI_COMMIT_REF_SLUG" ]; then + CI_COMMIT_REF_SLUG=$( + git rev-parse --abbrev-ref HEAD | tr "[:upper:]" "[:lower:]" \ + | LANG=en_US.utf8 sed -E -e 's/[^a-zA-Z0-9]/-/g' -e 's/^-+|-+$$//g' \ + | cut -c 1-63 + ) + fi + if [ -z "$CI_DEFAULT_BRANCH" ]; then + CI_DEFAULT_BRANCH="main" + fi +} + +print_help () { + echo "Usage: $(basename "$0") " +} + +parse_args() { + if [[ "$#" -ne 1 ]]; then + print_help + exit 0 + fi + + action="$1" + + valid_actions="(build|publish)" + valid_actions_ptn="^${valid_actions}$" + if ! [[ "$action" =~ $valid_actions_ptn ]]; then + echo "Invalid action. Valid actions: $valid_actions" + print_help + exit 1 + fi +} + +# MAIN +image="nlk" +parse_args "$@" +init_ci_vars + +# shellcheck source=/dev/null +source "${ROOT_DIR}/.devops.sh" +if [ "$CI" != "true" ]; then + devops.backend.docker.set "azure.container-registry-dev" +fi +repo="${DEVOPS_DOCKER_URL}/nginx-azure-lb/${CI_PROJECT_NAME}/$image" +# shellcheck source=/dev/null +# shellcheck disable=SC2153 +version=$(source "${ROOT_DIR}/version";echo "$VERSION") + +"$action" diff --git a/scripts/release-tag.sh b/scripts/release-tag.sh new file mode 100755 index 0000000..8c6d639 --- /dev/null +++ b/scripts/release-tag.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +### This script should only run during master pipelines ### +### This script will create tags for the master commit using the current version ### + +set -eo pipefail + +if [ "$CI_COMMIT_REF_NAME" = "main" ]; then + # shellcheck source=/dev/null + version_tag=v$(source version;echo "$VERSION") + + curl -s \ + --request POST \ + --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN_RW}" \ + "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/repository/tags" \ + --form "tag_name=$version_tag" \ + --form "ref=$CI_COMMIT_SHA" +fi diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..107cee4 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex + +export GO_DATA_RACE=${GO_DATA_RACE:-false} +if [ "$GO_DATA_RACE" == "true" ]; then + go_flags+=("-race") +fi + +outfile="${RESULTS_DIR}/coverage.out" +mkdir -p "$RESULTS_DIR" +go_flags+=("-cover" -coverprofile="$outfile") +gotestsum --junitfile "${RESULTS_DIR}/report.xml" --format pkgname -- "${go_flags[@]}" ./... +echo "Total code coverage:" +go tool cover -func="$outfile" | grep 'total:' | tee "${RESULTS_DIR}/anybadge.out" +go tool cover -html="$outfile" -o "${RESULTS_DIR}/coverage.html" diff --git a/version b/version new file mode 100644 index 0000000..e0212c0 --- /dev/null +++ b/version @@ -0,0 +1 @@ +export VERSION="1.$(date +"%Y%m%d").${CI_PIPELINE_ID:-0}" From f1804c4b0857d650e6ddc491684fee063e4e7500 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 27 Mar 2024 18:44:57 -0700 Subject: [PATCH 002/136] Remove Github-y stuff --- .github/CODEOWNERS | 3 - .github/ISSUE_TEMPLATE/bug_report.md | 32 ------- .github/ISSUE_TEMPLATE/feature_request.md | 22 ----- .github/dependabot.yml | 9 -- .github/pull_request_template.md | 12 --- .github/workflows/build-and-sign-image.yml | 98 ---------------------- .github/workflows/run-scorecard.yml | 72 ---------------- .github/workflows/run-tests.yml | 32 ------- 8 files changed, 280 deletions(-) delete mode 100644 .github/CODEOWNERS delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 .github/dependabot.yml delete mode 100644 .github/pull_request_template.md delete mode 100644 .github/workflows/build-and-sign-image.yml delete mode 100644 .github/workflows/run-scorecard.yml delete mode 100644 .github/workflows/run-tests.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index f330be6..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,3 +0,0 @@ -# Main global owner # -##################### -* @ciroque @chrisakker diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index cc6c1d2..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' ---- -### Describe the bug - -A clear and concise description of what the bug is. - -### To reproduce - -Steps to reproduce the behavior: - -1. Deploy nginx_loadbalancer_kubernetes using -2. View output/logs/configuration on '...' -3. See error - -### Expected behavior - -A clear and concise description of what you expected to happen. - -### Your environment - -- Version of the nginx_loadbalancer_kubernetes or specific commit - -- Target deployment platform - -### Additional context - -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index d27aba8..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' ---- -### Is your feature request related to a problem? Please describe - -A clear and concise description of what the problem is. Ex. I'm always frustrated when ... - -### Describe the solution you'd like - -A clear and concise description of what you want to happen. - -### Describe alternatives you've considered - -A clear and concise description of any alternative solutions or features you've considered. - -### Additional context - -Add any other context or screenshots about the feature request here. diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 4450376..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -version: 2 -updates: - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - day: monday - time: "00:00" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index fad5aa1..0000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,12 +0,0 @@ -### Proposed changes - -Describe the use case and detail of the change. If this PR addresses an issue on GitHub, make sure to include a link to that issue using one of the [supported keywords](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue) here in this description (not in the title of the PR). - -### Checklist - -Before creating a PR, run through this checklist and mark each as complete. - -- [ ] I have read the [`CONTRIBUTING`](https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/CONTRIBUTING.md) document -- [ ] If applicable, I have added tests that prove my fix is effective or that my feature works -- [ ] If applicable, I have checked that any relevant tests pass after adding my changes -- [ ] I have updated any relevant documentation ([`README.md`](https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/README.md) and [`CHANGELOG.md`](https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/CHANGELOG.md)) diff --git a/.github/workflows/build-and-sign-image.yml b/.github/workflows/build-and-sign-image.yml deleted file mode 100644 index 2fbf227..0000000 --- a/.github/workflows/build-and-sign-image.yml +++ /dev/null @@ -1,98 +0,0 @@ -# This workflow will build and push a signed Docker image - -name: Build and sign image - -on: - push: - tags: - - "v[0-9]+.[0-9]+.[0-9]+" -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build_and_sign_image: - runs-on: ubuntu-latest - permissions: - contents: write - packages: write - id-token: write - security-events: write - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: anchore/sbom-action@v0 - with: - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - output-file: ./nginx-loadbalancer-kubernetes-${{env.GITHUB_REF_NAME}}.spdx.json - registry-username: ${{ github.actor }} - registry-password: ${{ secrets.GITHUB_TOKEN }} - - - name: Install cosign - uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 #v3.0.2 - with: - cosign-release: 'v1.13.1' - - - name: Log into registry ${{ env.REGISTRY }} for ${{ github.actor }} - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build Docker Image - id: docker-build-and-push - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 - with: - context: . - file: ./Dockerfile - push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{github.run_number}} - - - name: Sign the published Docker images - env: - COSIGN_EXPERIMENTAL: "true" - # This step uses the identity token to provision an ephemeral certificate - # against the sigstore community Fulcio instance. - run: cosign sign "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.docker-build-and-push.outputs.digest }}" - - # NOTE: This runs statically against the latest tag in Docker Hub which was not produced by this workflow - # This should be updated once this workflow is fully implemented - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@91713af97dc80187565512baba96e4364e983601 # 0.16.0 - continue-on-error: true - with: - image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - format: 'sarif' - output: 'trivy-results-${{ inputs.image }}.sarif' - ignore-unfixed: 'true' - - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v2.2.11 - continue-on-error: true - with: - sarif_file: 'trivy-results-${{ inputs.image }}.sarif' - sha: ${{ github.sha }} - ref: ${{ github.ref }} - - - name: Generate Release - uses: ncipollo/release-action@v1 - with: - artifacts: | - trivy-results-${{ inputs.image }}.sarif - ./nginx-loadbalancer-kubernetes-${{env.GITHUB_REF_NAME}}.spdx.json - body: | - # Release ${{env.GITHUB_REF_NAME}} - ## Changelog - ${{ steps.meta.outputs.changelog }} - generateReleaseNotes: true - makeLatest: false - name: "${{env.GITHUB_REF_NAME}}" diff --git a/.github/workflows/run-scorecard.yml b/.github/workflows/run-scorecard.yml deleted file mode 100644 index 3bbad84..0000000 --- a/.github/workflows/run-scorecard.yml +++ /dev/null @@ -1,72 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. They are provided -# by a third-party and are governed by separate terms of service, privacy -# policy, and support documentation. - -name: Scorecard supply-chain security -on: - # For Branch-Protection check. Only the default branch is supported. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection - branch_protection_rule: - # To guarantee Maintained check is occasionally updated. See - # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained - schedule: - - cron: '15 14 * * 3' - push: - branches: [ "main" ] - -# Declare default permissions as read only. -permissions: read-all - -jobs: - analysis: - name: Scorecard analysis - runs-on: ubuntu-latest - permissions: - # Needed to upload the results to code-scanning dashboard. - security-events: write - # Needed to publish results and get a badge (see publish_results below). - id-token: write - # Uncomment the permissions below if installing in a private repository. - # contents: read - # actions: read - - steps: - - name: "Checkout code" - uses: actions/checkout@v4 # v3.1.0 - with: - persist-credentials: false - - - name: "Run analysis" - uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 - with: - results_file: results.sarif - results_format: sarif - # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: - # - you want to enable the Branch-Protection check on a *public* repository, or - # - you are installing Scorecard on a *private* repository - # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} - - # Public repositories: - # - Publish results to OpenSSF REST API for easy access by consumers - # - Allows the repository to include the Scorecard badge. - # - See https://github.com/ossf/scorecard-action#publishing-results. - # For private repositories: - # - `publish_results` will always be set to `false`, regardless - # of the value entered here. - publish_results: true - - # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF - # format to the repository Actions tab. - - name: "Upload artifact" - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 - with: - name: SARIF file - path: results.sarif - retention-days: 5 - - # Upload the results to GitHub's code scanning dashboard. - - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12 - with: - sarif_file: results.sarif diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml deleted file mode 100644 index 454c716..0000000 --- a/.github/workflows/run-tests.yml +++ /dev/null @@ -1,32 +0,0 @@ -# This workflow will build a golang project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go - -name: Run tests - -on: - branch_protection_rule: - types: - - created - - push: - branches: - - main - - * - -jobs: - - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: 1.19 - - - name: Build - run: go build -v ./... - - - name: Test - run: go test -v ./... From d4f0824018e4ced9be31c67c49ab9955147205f1 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 29 Mar 2024 12:12:35 -0700 Subject: [PATCH 003/136] Format code --- cmd/certificates-test-harness/main.go | 3 ++- cmd/configuration-test-harness/main.go | 3 ++- cmd/nginx-loadbalancer-kubernetes/main.go | 1 + cmd/tls-config-factory-test-harness/main.go | 3 ++- internal/application/application_common_test.go | 1 + internal/application/border_client.go | 1 + internal/application/border_client_test.go | 3 ++- internal/application/nginx_http_border_client.go | 1 + internal/application/nginx_stream_border_client.go | 1 + internal/authentication/factory.go | 1 + internal/certification/certificates_test.go | 5 +++-- internal/communication/factory.go | 5 +++-- internal/communication/factory_test.go | 3 ++- internal/communication/roundtripper_test.go | 5 +++-- internal/configuration/settings.go | 5 +++-- internal/core/event_test.go | 3 ++- internal/observation/handler.go | 1 + internal/observation/handler_test.go | 3 ++- internal/observation/watcher.go | 3 ++- internal/observation/watcher_test.go | 3 ++- internal/probation/server.go | 3 ++- internal/probation/server_test.go | 5 +++-- internal/synchronization/synchronizer.go | 1 + internal/synchronization/synchronizer_test.go | 3 ++- internal/translation/translator.go | 3 ++- internal/translation/translator_test.go | 7 ++++--- 26 files changed, 51 insertions(+), 25 deletions(-) diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go index 44d4a4e..d2b746b 100644 --- a/cmd/certificates-test-harness/main.go +++ b/cmd/certificates-test-harness/main.go @@ -4,13 +4,14 @@ import ( "context" "errors" "fmt" + "path/filepath" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" - "path/filepath" ) func main() { diff --git a/cmd/configuration-test-harness/main.go b/cmd/configuration-test-harness/main.go index 56e8b5d..5079a9d 100644 --- a/cmd/configuration-test-harness/main.go +++ b/cmd/configuration-test-harness/main.go @@ -4,13 +4,14 @@ import ( "context" "errors" "fmt" + "path/filepath" + configuration2 "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" - "path/filepath" ) func main() { diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 6936557..0731965 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -8,6 +8,7 @@ package main import ( "context" "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/observation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/probation" diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 3f46d4f..1813bd9 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -3,12 +3,13 @@ package main import ( "bufio" "fmt" + "os" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" - "os" ) const ( diff --git a/internal/application/application_common_test.go b/internal/application/application_common_test.go index e963d03..f7e9561 100644 --- a/internal/application/application_common_test.go +++ b/internal/application/application_common_test.go @@ -7,6 +7,7 @@ package application import ( "errors" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" ) diff --git a/internal/application/border_client.go b/internal/application/border_client.go index a5cc93e..0ab6be6 100644 --- a/internal/application/border_client.go +++ b/internal/application/border_client.go @@ -7,6 +7,7 @@ package application import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" ) diff --git a/internal/application/border_client_test.go b/internal/application/border_client_test.go index 0b8105e..60ac41f 100644 --- a/internal/application/border_client_test.go +++ b/internal/application/border_client_test.go @@ -6,8 +6,9 @@ package application import ( - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" ) func TestBorderClient_CreatesHttpBorderClient(t *testing.T) { diff --git a/internal/application/nginx_http_border_client.go b/internal/application/nginx_http_border_client.go index b7657c5..78db773 100644 --- a/internal/application/nginx_http_border_client.go +++ b/internal/application/nginx_http_border_client.go @@ -7,6 +7,7 @@ package application import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index 46cd498..a0adff0 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -7,6 +7,7 @@ package application import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go index 8b8d06e..69c5ee7 100644 --- a/internal/authentication/factory.go +++ b/internal/authentication/factory.go @@ -12,6 +12,7 @@ import ( "crypto/x509" "encoding/pem" "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/sirupsen/logrus" diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go index c8edf14..d9d61ac 100644 --- a/internal/certification/certificates_test.go +++ b/internal/certification/certificates_test.go @@ -7,12 +7,13 @@ package certification import ( "context" + "testing" + "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/cache" - "testing" - "time" ) const ( diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 9a3d411..8b19490 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -7,11 +7,12 @@ package communication import ( "crypto/tls" + netHttp "net/http" + "time" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/sirupsen/logrus" - netHttp "net/http" - "time" ) // NewHttpClient is a factory method to create a new Http Client with a default configuration. diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index f25abef..f95722d 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -7,9 +7,10 @@ package communication import ( "context" + "testing" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "k8s.io/client-go/kubernetes/fake" - "testing" ) func TestNewHttpClient(t *testing.T) { diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index f46fb71..0a549d6 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -8,10 +8,11 @@ package communication import ( "bytes" "context" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "k8s.io/client-go/kubernetes/fake" netHttp "net/http" "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + "k8s.io/client-go/kubernetes/fake" ) func TestNewRoundTripper(t *testing.T) { diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index d15ad84..e6bd30b 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -8,6 +8,9 @@ package configuration import ( "context" "fmt" + "strings" + "time" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" @@ -16,8 +19,6 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "strings" - "time" ) const ( diff --git a/internal/core/event_test.go b/internal/core/event_test.go index b3b8926..7f7448d 100644 --- a/internal/core/event_test.go +++ b/internal/core/event_test.go @@ -1,8 +1,9 @@ package core import ( - v1 "k8s.io/api/core/v1" "testing" + + v1 "k8s.io/api/core/v1" ) func TestNewEvent(t *testing.T) { diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 83601b0..8e89745 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -7,6 +7,7 @@ package observation import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index f4a617f..e5bef3d 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -8,12 +8,13 @@ package observation import ( "context" "fmt" + "testing" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" v1 "k8s.io/api/core/v1" "k8s.io/client-go/util/workqueue" - "testing" ) func TestHandler_AddsEventToSynchronizer(t *testing.T) { diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 3ee9d3f..74e2d05 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -8,6 +8,8 @@ package observation import ( "errors" "fmt" + "time" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" @@ -16,7 +18,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" "k8s.io/client-go/tools/cache" - "time" ) // Watcher is responsible for watching for changes to Kubernetes resources. diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 36d64ee..7c820bf 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -7,10 +7,11 @@ package observation import ( "context" + "testing" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" "k8s.io/client-go/kubernetes" - "testing" ) func TestWatcher_MustInitialize(t *testing.T) { diff --git a/internal/probation/server.go b/internal/probation/server.go index 12b1699..ff23e33 100644 --- a/internal/probation/server.go +++ b/internal/probation/server.go @@ -7,8 +7,9 @@ package probation import ( "fmt" - "github.com/sirupsen/logrus" "net/http" + + "github.com/sirupsen/logrus" ) const ( diff --git a/internal/probation/server_test.go b/internal/probation/server_test.go index f594bff..4ea51e7 100644 --- a/internal/probation/server_test.go +++ b/internal/probation/server_test.go @@ -6,10 +6,11 @@ package probation import ( - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "github.com/sirupsen/logrus" "net/http" "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" + "github.com/sirupsen/logrus" ) func TestHealthServer_HandleLive(t *testing.T) { diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 1061b01..5fc07f5 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -7,6 +7,7 @@ package synchronization import ( "fmt" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/communication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index 315def7..ef510b8 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -8,10 +8,11 @@ package synchronization import ( "context" "fmt" + "testing" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "testing" ) func TestSynchronizer_NewSynchronizer(t *testing.T) { diff --git a/internal/translation/translator.go b/internal/translation/translator.go index b2d0e87..98ff368 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -7,12 +7,13 @@ package translation import ( "fmt" + "strings" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" - "strings" ) // Translate transforms event data into an intermediate format that can be consumed by the BorderClient implementations diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index 2acfd34..a393f64 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -7,12 +7,13 @@ package translation import ( "fmt" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - v1 "k8s.io/api/core/v1" "math/rand" "testing" "time" + + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" + v1 "k8s.io/api/core/v1" ) const ( From 12b2dd0083b3b1c440f2c1775e0c6907da43493f Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 29 Mar 2024 12:33:13 -0700 Subject: [PATCH 004/136] Lint code --- cmd/certificates-test-harness/doc.go | 3 +- cmd/certificates-test-harness/main.go | 2 +- cmd/nginx-loadbalancer-kubernetes/main.go | 14 ++---- cmd/tls-config-factory-test-harness/main.go | 45 ++++++++--------- doc.go | 2 +- .../application/application_common_test.go | 4 +- internal/application/application_constants.go | 7 +-- internal/application/border_client.go | 4 +- internal/application/border_client_test.go | 7 ++- internal/application/doc.go | 5 +- .../application/nginx_client_interface.go | 17 +++++-- .../application/nginx_http_border_client.go | 27 ++++++----- .../nginx_http_border_client_test.go | 26 ++++++---- .../application/nginx_stream_border_client.go | 5 +- .../nginx_stream_border_client_test.go | 12 +++-- internal/application/null_border_client.go | 3 +- .../application/null_border_client_test.go | 2 + internal/authentication/factory.go | 31 ++++++------ internal/authentication/factory_test.go | 48 +++++++++++-------- internal/certification/certificates.go | 12 ++--- internal/certification/certificates_test.go | 6 +++ internal/communication/factory.go | 14 +++--- internal/communication/factory_test.go | 12 +++-- internal/communication/roundtripper.go | 3 +- internal/communication/roundtripper_test.go | 10 +++- internal/configuration/settings.go | 38 ++++++++------- internal/configuration/tlsmodes_test.go | 2 + internal/core/event_test.go | 1 + internal/core/secret_bytes_test.go | 2 + internal/core/server_update_event.go | 15 ++++-- internal/core/server_update_event_test.go | 23 +++++---- internal/core/upstream_server.go | 3 +- internal/core/upstream_server_test.go | 1 + internal/observation/handler.go | 6 ++- internal/observation/handler_test.go | 7 ++- internal/observation/watcher.go | 28 ++++++----- internal/observation/watcher_test.go | 1 + internal/probation/check_test.go | 3 ++ internal/probation/server.go | 3 +- internal/probation/server_test.go | 6 +++ internal/synchronization/rand.go | 5 +- internal/synchronization/synchronizer.go | 31 +++++++----- internal/synchronization/synchronizer_test.go | 32 ++++++++++++- internal/translation/translator.go | 24 ++++++---- internal/translation/translator_test.go | 34 ++++++++++++- test/mocks/mock_nginx_plus_client.go | 10 +++- 46 files changed, 388 insertions(+), 208 deletions(-) diff --git a/cmd/certificates-test-harness/doc.go b/cmd/certificates-test-harness/doc.go index 538bed9..2d76fd5 100644 --- a/cmd/certificates-test-harness/doc.go +++ b/cmd/certificates-test-harness/doc.go @@ -4,7 +4,8 @@ */ /* -Package certificates_test_harness includes functionality boostrap and test the certification.Certificates implplementation. +Package certificates_test_harness includes functionality boostrap +and test the certification.Certificates implplementation. */ package main diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go index d2b746b..056d303 100644 --- a/cmd/certificates-test-harness/main.go +++ b/cmd/certificates-test-harness/main.go @@ -40,7 +40,7 @@ func run() error { return fmt.Errorf(`error occurred initializing certificates: %w`, err) } - go certificates.Run() + go certificates.Run() //nolint:errcheck <-ctx.Done() return nil diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 0731965..89f764f 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -47,20 +47,14 @@ func run() error { go settings.Run() - synchronizerWorkqueue, err := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - if err != nil { - return fmt.Errorf(`error occurred building a workqueue: %w`, err) - } + synchronizerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) synchronizer, err := synchronization.NewSynchronizer(settings, synchronizerWorkqueue) if err != nil { return fmt.Errorf(`error initializing synchronizer: %w`, err) } - handlerWorkqueue, err := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - if err != nil { - return fmt.Errorf(`error occurred building a workqueue: %w`, err) - } + handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue) @@ -106,9 +100,9 @@ func buildKubernetesClient() (*kubernetes.Clientset, error) { return client, nil } -func buildWorkQueue(settings configuration.WorkQueueSettings) (workqueue.RateLimitingInterface, error) { +func buildWorkQueue(settings configuration.WorkQueueSettings) workqueue.RateLimitingInterface { logrus.Debug("Watcher::buildSynchronizerWorkQueue") rateLimiter := workqueue.NewItemExponentialFailureRateLimiter(settings.RateLimiterBase, settings.RateLimiterMax) - return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name), nil + return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name) } diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 1813bd9..51f1d1d 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "fmt" "os" "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" @@ -17,7 +16,7 @@ const ( ClientCertificateSecretKey = "nlk-tls-client-secret" ) -type TlsConfiguration struct { +type TLSConfiguration struct { Description string Settings configuration.Settings } @@ -28,11 +27,11 @@ func main() { configurations := buildConfigMap() for name, settings := range configurations { - fmt.Print("\033[H\033[2J") + logrus.Infof("\033[H\033[2J") logrus.Infof("\n\n\t*** Building TLS config for <<< %s >>>\n\n", name) - tlsConfig, err := authentication.NewTlsConfig(&settings.Settings) + tlsConfig, err := authentication.NewTLSConfig(&settings.Settings) if err != nil { panic(err) } @@ -41,41 +40,43 @@ func main() { certificateCount := 0 if tlsConfig.RootCAs != nil { - rootCaCount = len(tlsConfig.RootCAs.Subjects()) + rootCaCount = len(tlsConfig.RootCAs.Subjects()) //nolint:staticcheck } if tlsConfig.Certificates != nil { certificateCount = len(tlsConfig.Certificates) } - logrus.Infof("Successfully built TLS config: \n\tDescription: %s \n\tRootCA count: %v\n\tCertificate count: %v", settings.Description, rootCaCount, certificateCount) + logrus.Infof("Successfully built TLS config: \n\tDescription: %s \n\tRootCA count: %v\n\tCertificate count: %v", + settings.Description, rootCaCount, certificateCount, + ) - bufio.NewReader(os.Stdin).ReadBytes('\n') + _, _ = bufio.NewReader(os.Stdin).ReadBytes('\n') } - fmt.Print("\033[H\033[2J") + logrus.Infof("\033[H\033[2J") logrus.Infof("\n\n\t*** All done! ***\n\n") } -func buildConfigMap() map[string]TlsConfiguration { - configurations := make(map[string]TlsConfiguration) +func buildConfigMap() map[string]TLSConfiguration { + configurations := make(map[string]TLSConfiguration) - configurations["ss-tls"] = TlsConfiguration{ + configurations["ss-tls"] = TLSConfiguration{ Description: "Self-signed TLS requires just a CA certificate", - Settings: ssTlsConfig(), + Settings: ssTLSConfig(), } - configurations["ss-mtls"] = TlsConfiguration{ + configurations["ss-mtls"] = TLSConfiguration{ Description: "Self-signed mTLS requires a CA certificate and a client certificate", Settings: ssMtlsConfig(), } - configurations["ca-tls"] = TlsConfiguration{ + configurations["ca-tls"] = TLSConfiguration{ Description: "CA TLS requires no certificates", - Settings: caTlsConfig(), + Settings: caTLSConfig(), } - configurations["ca-mtls"] = TlsConfiguration{ + configurations["ca-mtls"] = TLSConfiguration{ Description: "CA mTLS requires a client certificate", Settings: caMtlsConfig(), } @@ -83,13 +84,13 @@ func buildConfigMap() map[string]TlsConfiguration { return configurations } -func ssTlsConfig() configuration.Settings { +func ssTLSConfig() configuration.Settings { certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) return configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, + TLSMode: configuration.SelfSignedTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, @@ -102,16 +103,16 @@ func ssMtlsConfig() configuration.Settings { certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) return configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, + TLSMode: configuration.SelfSignedMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, } } -func caTlsConfig() configuration.Settings { +func caTLSConfig() configuration.Settings { return configuration.Settings{ - TlsMode: configuration.CertificateAuthorityTLS, + TLSMode: configuration.CertificateAuthorityTLS, } } @@ -120,7 +121,7 @@ func caMtlsConfig() configuration.Settings { certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) return configuration.Settings{ - TlsMode: configuration.CertificateAuthorityMutualTLS, + TLSMode: configuration.CertificateAuthorityMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, diff --git a/doc.go b/doc.go index 7c97bd2..1034f14 100644 --- a/doc.go +++ b/doc.go @@ -3,4 +3,4 @@ * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ -package kubernetes_nginx_ingress +package kubernetesnginxingress diff --git a/internal/application/application_common_test.go b/internal/application/application_common_test.go index f7e9561..c42bc04 100644 --- a/internal/application/application_common_test.go +++ b/internal/application/application_common_test.go @@ -19,11 +19,11 @@ const ( server = "server" ) -func buildTerrorizingBorderClient(clientType string) (Interface, *mocks.MockNginxClient, error) { +func buildTerrorizingBorderClient(clientType string) (Interface, error) { nginxClient := mocks.NewErroringMockClient(errors.New(`something went horribly horribly wrong`)) bc, err := NewBorderClient(clientType, nginxClient) - return bc, nginxClient, err + return bc, err } func buildBorderClient(clientType string) (Interface, *mocks.MockNginxClient, error) { diff --git a/internal/application/application_constants.go b/internal/application/application_constants.go index 0ec1826..4cb23a5 100644 --- a/internal/application/application_constants.go +++ b/internal/application/application_constants.go @@ -12,7 +12,8 @@ package application // annotations: // nginxinc.io/nlk-: // -// where is the name of the upstream in the NGINX Plus configuration and is one of the constants below. +// where is the name of the upstream in the NGINX Plus configuration +// and is one of the constants below. // // Note, this is an extensibility point. To add a Border Server client... // 1. Create a module that implements the BorderClient interface; @@ -23,6 +24,6 @@ const ( // ClientTypeNginxStream creates a NginxStreamBorderClient that uses the Stream* methods of the NGINX Plus client. ClientTypeNginxStream = "stream" - // ClientTypeNginxHttp creates an NginxHttpBorderClient that uses the HTTP* methods of the NGINX Plus client. - ClientTypeNginxHttp = "http" + // ClientTypeNginxHTTP creates an NginxHTTPBorderClient that uses the HTTP* methods of the NGINX Plus client. + ClientTypeNginxHTTP = "http" ) diff --git a/internal/application/border_client.go b/internal/application/border_client.go index 0ab6be6..a510953 100644 --- a/internal/application/border_client.go +++ b/internal/application/border_client.go @@ -35,8 +35,8 @@ func NewBorderClient(clientType string, borderClient interface{}) (Interface, er case ClientTypeNginxStream: return NewNginxStreamBorderClient(borderClient) - case ClientTypeNginxHttp: - return NewNginxHttpBorderClient(borderClient) + case ClientTypeNginxHTTP: + return NewNginxHTTPBorderClient(borderClient) default: borderClient, _ := NewNullBorderClient() diff --git a/internal/application/border_client_test.go b/internal/application/border_client_test.go index 60ac41f..8960eee 100644 --- a/internal/application/border_client_test.go +++ b/internal/application/border_client_test.go @@ -12,18 +12,20 @@ import ( ) func TestBorderClient_CreatesHttpBorderClient(t *testing.T) { + t.Parallel() borderClient := mocks.MockNginxClient{} client, err := NewBorderClient("http", borderClient) if err != nil { t.Errorf(`error creating border client: %v`, err) } - if _, ok := client.(*NginxHttpBorderClient); !ok { - t.Errorf(`expected client to be of type NginxHttpBorderClient`) + if _, ok := client.(*NginxHTTPBorderClient); !ok { + t.Errorf(`expected client to be of type NginxHTTPBorderClient`) } } func TestBorderClient_CreatesTcpBorderClient(t *testing.T) { + t.Parallel() borderClient := mocks.MockNginxClient{} client, err := NewBorderClient("stream", borderClient) if err != nil { @@ -36,6 +38,7 @@ func TestBorderClient_CreatesTcpBorderClient(t *testing.T) { } func TestBorderClient_UnknownClientType(t *testing.T) { + t.Parallel() unknownClientType := "unknown" borderClient := mocks.MockNginxClient{} client, err := NewBorderClient(unknownClientType, borderClient) diff --git a/internal/application/doc.go b/internal/application/doc.go index 34c27d0..296cb67 100644 --- a/internal/application/doc.go +++ b/internal/application/doc.go @@ -17,7 +17,7 @@ To add a Border Server client... At this time the only supported Border Servers are NGINX Plus servers. The two Border Server clients for NGINX Plus are: -- NginxHttpBorderClient: updates NGINX Plus servers using HTTP Upstream methods on the NGINX Plus API. +- NginxHTTPBorderClient: updates NGINX Plus servers using HTTP Upstream methods on the NGINX Plus API. - NginxStreamBorderClient: updates NGINX Plus servers using Stream Upstream methods on the NGINX Plus API. Both of these implementations use the NGINX Plus client module to communicate with the NGINX Plus server. @@ -27,7 +27,8 @@ Selection of the appropriate client is based on the Annotations present on the S annotations: nginxinc.io/nlk-: -where is the name of the upstream in the NGINX Plus configuration and is one of the constants in application_constants.go. +where is the name of the upstream in the NGINX Plus configuration +and is one of the constants in application_constants.go. */ package application diff --git a/internal/application/nginx_client_interface.go b/internal/application/nginx_client_interface.go index 728db1e..31a7b94 100644 --- a/internal/application/nginx_client_interface.go +++ b/internal/application/nginx_client_interface.go @@ -7,17 +7,24 @@ package application import nginxClient "github.com/nginxinc/nginx-plus-go-client/client" -// NginxClientInterface defines the functions used on the NGINX Plus client, abstracting away the full details of that client. +// NginxClientInterface defines the functions used on the NGINX Plus client, +// abstracting away the full details of that client. type NginxClientInterface interface { // DeleteStreamServer is used by the NginxStreamBorderClient. DeleteStreamServer(upstream string, server string) error // UpdateStreamServers is used by the NginxStreamBorderClient. - UpdateStreamServers(upstream string, servers []nginxClient.StreamUpstreamServer) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) + UpdateStreamServers( + upstream string, + servers []nginxClient.StreamUpstreamServer, + ) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) - // DeleteHTTPServer is used by the NginxHttpBorderClient. + // DeleteHTTPServer is used by the NginxHTTPBorderClient. DeleteHTTPServer(upstream string, server string) error - // UpdateHTTPServers is used by the NginxHttpBorderClient. - UpdateHTTPServers(upstream string, servers []nginxClient.UpstreamServer) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) + // UpdateHTTPServers is used by the NginxHTTPBorderClient. + UpdateHTTPServers( + upstream string, + servers []nginxClient.UpstreamServer, + ) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) } diff --git a/internal/application/nginx_http_border_client.go b/internal/application/nginx_http_border_client.go index 78db773..e9e754a 100644 --- a/internal/application/nginx_http_border_client.go +++ b/internal/application/nginx_http_border_client.go @@ -2,7 +2,8 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ - +// dupl complains about duplicates with nginx_stream_border_client.go +//nolint:dupl package application import ( @@ -13,26 +14,26 @@ import ( ) // NginxHttpBorderClient implements the BorderClient interface for HTTP upstreams. -type NginxHttpBorderClient struct { +type NginxHTTPBorderClient struct { BorderClient nginxClient NginxClientInterface } -// NewNginxHttpBorderClient is the Factory function for creating an NginxHttpBorderClient. -func NewNginxHttpBorderClient(client interface{}) (Interface, error) { +// NewNginxHTTPBorderClient is the Factory function for creating an NewNginxHTTPBorderClient. +func NewNginxHTTPBorderClient(client interface{}) (Interface, error) { ngxClient, ok := client.(NginxClientInterface) if !ok { return nil, fmt.Errorf(`expected a NginxClientInterface, got a %v`, client) } - return &NginxHttpBorderClient{ + return &NginxHTTPBorderClient{ nginxClient: ngxClient, }, nil } // Update manages the Upstream servers for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHttpBorderClient) Update(event *core.ServerUpdateEvent) error { - httpUpstreamServers := asNginxHttpUpstreamServers(event.UpstreamServers) +func (hbc *NginxHTTPBorderClient) Update(event *core.ServerUpdateEvent) error { + httpUpstreamServers := asNginxHTTPUpstreamServers(event.UpstreamServers) _, _, _, err := hbc.nginxClient.UpdateHTTPServers(event.UpstreamName, httpUpstreamServers) if err != nil { return fmt.Errorf(`error occurred updating the nginx+ upstream server: %w`, err) @@ -42,7 +43,7 @@ func (hbc *NginxHttpBorderClient) Update(event *core.ServerUpdateEvent) error { } // Delete deletes the Upstream server for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHttpBorderClient) Delete(event *core.ServerUpdateEvent) error { +func (hbc *NginxHTTPBorderClient) Delete(event *core.ServerUpdateEvent) error { err := hbc.nginxClient.DeleteHTTPServer(event.UpstreamName, event.UpstreamServers[0].Host) if err != nil { return fmt.Errorf(`error occurred deleting the nginx+ upstream server: %w`, err) @@ -52,18 +53,18 @@ func (hbc *NginxHttpBorderClient) Delete(event *core.ServerUpdateEvent) error { } // asNginxHttpUpstreamServer converts a core.UpstreamServer to a nginxClient.UpstreamServer. -func asNginxHttpUpstreamServer(server *core.UpstreamServer) nginxClient.UpstreamServer { +func asNginxHTTPUpstreamServer(server *core.UpstreamServer) nginxClient.UpstreamServer { return nginxClient.UpstreamServer{ Server: server.Host, } } -// asNginxHttpUpstreamServers converts a core.UpstreamServers to a []nginxClient.UpstreamServer. -func asNginxHttpUpstreamServers(servers core.UpstreamServers) []nginxClient.UpstreamServer { - var upstreamServers []nginxClient.UpstreamServer +// asNginxHTTPUpstreamServers converts a core.UpstreamServers to a []nginxClient.UpstreamServer. +func asNginxHTTPUpstreamServers(servers core.UpstreamServers) []nginxClient.UpstreamServer { + upstreamServers := []nginxClient.UpstreamServer{} for _, server := range servers { - upstreamServers = append(upstreamServers, asNginxHttpUpstreamServer(server)) + upstreamServers = append(upstreamServers, asNginxHTTPUpstreamServer(server)) } return upstreamServers diff --git a/internal/application/nginx_http_border_client_test.go b/internal/application/nginx_http_border_client_test.go index defc2ef..039b4ec 100644 --- a/internal/application/nginx_http_border_client_test.go +++ b/internal/application/nginx_http_border_client_test.go @@ -3,6 +3,9 @@ * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ +// dupl complains about duplicates with nginx_stream_border_client_test.go +// +//nolint:dupl package application import ( @@ -10,8 +13,9 @@ import ( ) func TestHttpBorderClient_Delete(t *testing.T) { - event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHttp) - borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHTTP) + borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } @@ -27,8 +31,9 @@ func TestHttpBorderClient_Delete(t *testing.T) { } func TestHttpBorderClient_Update(t *testing.T) { - event := buildServerUpdateEvent(createEventType, ClientTypeNginxHttp) - borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(createEventType, ClientTypeNginxHTTP) + borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } @@ -44,16 +49,18 @@ func TestHttpBorderClient_Update(t *testing.T) { } func TestHttpBorderClient_BadNginxClient(t *testing.T) { + t.Parallel() var emptyInterface interface{} - _, err := NewBorderClient(ClientTypeNginxHttp, emptyInterface) + _, err := NewBorderClient(ClientTypeNginxHTTP, emptyInterface) if err == nil { t.Fatalf(`expected an error to occur when creating a new border client`) } } func TestHttpBorderClient_DeleteReturnsError(t *testing.T) { - event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHttp) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxHTTP) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } @@ -66,8 +73,9 @@ func TestHttpBorderClient_DeleteReturnsError(t *testing.T) { } func TestHttpBorderClient_UpdateReturnsError(t *testing.T) { - event := buildServerUpdateEvent(createEventType, ClientTypeNginxHttp) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxHttp) + t.Parallel() + event := buildServerUpdateEvent(createEventType, ClientTypeNginxHTTP) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxHTTP) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index a0adff0..19982b4 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -2,7 +2,8 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ - +// dupl complains about duplicates with nginx_http_border_client.go +//nolint:dupl package application import ( @@ -58,7 +59,7 @@ func asNginxStreamUpstreamServer(server *core.UpstreamServer) nginxClient.Stream } func asNginxStreamUpstreamServers(servers core.UpstreamServers) []nginxClient.StreamUpstreamServer { - var upstreamServers []nginxClient.StreamUpstreamServer + upstreamServers := []nginxClient.StreamUpstreamServer{} for _, server := range servers { upstreamServers = append(upstreamServers, asNginxStreamUpstreamServer(server)) diff --git a/internal/application/nginx_stream_border_client_test.go b/internal/application/nginx_stream_border_client_test.go index ddcb346..c86a776 100644 --- a/internal/application/nginx_stream_border_client_test.go +++ b/internal/application/nginx_stream_border_client_test.go @@ -2,7 +2,8 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. */ - +// dupl complains about duplicates with nginx_http_border_client_test.go +//nolint:dupl package application import ( @@ -10,6 +11,7 @@ import ( ) func TestTcpBorderClient_Delete(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxStream) borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxStream) if err != nil { @@ -27,6 +29,7 @@ func TestTcpBorderClient_Delete(t *testing.T) { } func TestTcpBorderClient_Update(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(createEventType, ClientTypeNginxStream) borderClient, nginxClient, err := buildBorderClient(ClientTypeNginxStream) if err != nil { @@ -44,6 +47,7 @@ func TestTcpBorderClient_Update(t *testing.T) { } func TestTcpBorderClient_BadNginxClient(t *testing.T) { + t.Parallel() var emptyInterface interface{} _, err := NewBorderClient(ClientTypeNginxStream, emptyInterface) if err == nil { @@ -52,8 +56,9 @@ func TestTcpBorderClient_BadNginxClient(t *testing.T) { } func TestTcpBorderClient_DeleteReturnsError(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(deletedEventType, ClientTypeNginxStream) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxStream) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxStream) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } @@ -66,8 +71,9 @@ func TestTcpBorderClient_DeleteReturnsError(t *testing.T) { } func TestTcpBorderClient_UpdateReturnsError(t *testing.T) { + t.Parallel() event := buildServerUpdateEvent(createEventType, ClientTypeNginxStream) - borderClient, _, err := buildTerrorizingBorderClient(ClientTypeNginxStream) + borderClient, err := buildTerrorizingBorderClient(ClientTypeNginxStream) if err != nil { t.Fatalf(`error occurred creating a new border client: %v`, err) } diff --git a/internal/application/null_border_client.go b/internal/application/null_border_client.go index 8370fe0..b59ca22 100644 --- a/internal/application/null_border_client.go +++ b/internal/application/null_border_client.go @@ -11,7 +11,8 @@ import ( ) // NullBorderClient is a BorderClient that does nothing. -// / It serves only to prevent a panic if the BorderClient is not set correctly and errors from the factory methods are ignored. +// It serves only to prevent a panic if the BorderClient +// is not set correctly and errors from the factory methods are ignored. type NullBorderClient struct { } diff --git a/internal/application/null_border_client_test.go b/internal/application/null_border_client_test.go index 42e9dfb..f973949 100644 --- a/internal/application/null_border_client_test.go +++ b/internal/application/null_border_client_test.go @@ -8,6 +8,7 @@ package application import "testing" func TestNullBorderClient_Delete(t *testing.T) { + t.Parallel() client := NullBorderClient{} err := client.Delete(nil) if err != nil { @@ -16,6 +17,7 @@ func TestNullBorderClient_Delete(t *testing.T) { } func TestNullBorderClient_Update(t *testing.T) { + t.Parallel() client := NullBorderClient{} err := client.Update(nil) if err != nil { diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go index 69c5ee7..32add62 100644 --- a/internal/authentication/factory.go +++ b/internal/authentication/factory.go @@ -18,37 +18,38 @@ import ( "github.com/sirupsen/logrus" ) -func NewTlsConfig(settings *configuration.Settings) (*tls.Config, error) { - logrus.Debugf("authentication::NewTlsConfig Creating TLS config for mode: '%s'", settings.TlsMode) - switch settings.TlsMode { +func NewTLSConfig(settings *configuration.Settings) (*tls.Config, error) { + logrus.Debugf("authentication::NewTLSConfig Creating TLS config for mode: '%s'", settings.TLSMode) + switch settings.TLSMode { case configuration.NoTLS: - return buildBasicTlsConfig(true), nil + return buildBasicTLSConfig(true), nil case configuration.SelfSignedTLS: // needs ca cert - return buildSelfSignedTlsConfig(settings.Certificates) + return buildSelfSignedTLSConfig(settings.Certificates) case configuration.SelfSignedMutualTLS: // needs ca cert and client cert return buildSelfSignedMtlsConfig(settings.Certificates) case configuration.CertificateAuthorityTLS: // needs nothing - return buildBasicTlsConfig(false), nil + return buildBasicTLSConfig(false), nil case configuration.CertificateAuthorityMutualTLS: // needs client cert - return buildCaTlsConfig(settings.Certificates) + return buildCATLSConfig(settings.Certificates) default: - return nil, fmt.Errorf("unknown TLS mode: %s", settings.TlsMode) + return nil, fmt.Errorf("unknown TLS mode: %s", settings.TLSMode) } } -func buildSelfSignedTlsConfig(certificates *certification.Certificates) (*tls.Config, error) { +func buildSelfSignedTLSConfig(certificates *certification.Certificates) (*tls.Config, error) { logrus.Debug("authentication::buildSelfSignedTlsConfig Building self-signed TLS config") certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) if err != nil { return nil, err } + //nolint:gosec return &tls.Config{ InsecureSkipVerify: false, RootCAs: certPool, @@ -68,6 +69,7 @@ func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.C } logrus.Debugf("buildSelfSignedMtlsConfig Certificate: %v", certificate) + //nolint:gosec return &tls.Config{ InsecureSkipVerify: false, RootCAs: certPool, @@ -76,20 +78,21 @@ func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.C }, nil } -func buildBasicTlsConfig(skipVerify bool) *tls.Config { - logrus.Debugf("authentication::buildBasicTlsConfig skipVerify(%v)", skipVerify) +func buildBasicTLSConfig(skipVerify bool) *tls.Config { + logrus.Debugf("authentication::buildBasicTLSConfig skipVerify(%v)", skipVerify) return &tls.Config{ - InsecureSkipVerify: skipVerify, + InsecureSkipVerify: skipVerify, //nolint:gosec } } -func buildCaTlsConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildCaTlsConfig") +func buildCATLSConfig(certificates *certification.Certificates) (*tls.Config, error) { + logrus.Debug("authentication::buildCATLSConfig") certificate, err := buildCertificates(certificates.GetClientCertificate()) if err != nil { return nil, err } + //nolint:gosec return &tls.Config{ InsecureSkipVerify: false, Certificates: []tls.Certificate{certificate}, diff --git a/internal/authentication/factory_test.go b/internal/authentication/factory_test.go index a535200..e9015c0 100644 --- a/internal/authentication/factory_test.go +++ b/internal/authentication/factory_test.go @@ -19,9 +19,10 @@ const ( ) func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { + t.Parallel() settings := configuration.Settings{} - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -36,11 +37,12 @@ func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { } func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, + TLSMode: configuration.SelfSignedTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -48,7 +50,7 @@ func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { }, } - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -71,17 +73,18 @@ func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { } func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, + TLSMode: configuration.SelfSignedTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -92,11 +95,12 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { } func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificateDataPEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedTLS, + TLSMode: configuration.SelfSignedTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -104,7 +108,7 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -115,12 +119,13 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) } func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, + TLSMode: configuration.SelfSignedMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -128,7 +133,7 @@ func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { }, } - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -151,18 +156,19 @@ func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { } func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, + TLSMode: configuration.SelfSignedMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -173,12 +179,13 @@ func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { } func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.SelfSignedMutualTLS, + TLSMode: configuration.SelfSignedMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -186,7 +193,7 @@ func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -197,11 +204,12 @@ func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { } func TestTlsFactory_CaTlsMode(t *testing.T) { + t.Parallel() settings := configuration.Settings{ - TlsMode: configuration.CertificateAuthorityTLS, + TLSMode: configuration.CertificateAuthorityTLS, } - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -224,11 +232,12 @@ func TestTlsFactory_CaTlsMode(t *testing.T) { } func TestTlsFactory_CaMtlsMode(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.CertificateAuthorityMutualTLS, + TLSMode: configuration.CertificateAuthorityMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, CaCertificateSecretKey: CaCertificateSecretKey, @@ -236,7 +245,7 @@ func TestTlsFactory_CaMtlsMode(t *testing.T) { }, } - tlsConfig, err := NewTlsConfig(&settings) + tlsConfig, err := NewTLSConfig(&settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -259,18 +268,19 @@ func TestTlsFactory_CaMtlsMode(t *testing.T) { } func TestTlsFactory_CaMtlsModeClientCertificateError(t *testing.T) { + t.Parallel() certificates := make(map[string]map[string]core.SecretBytes) certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) settings := configuration.Settings{ - TlsMode: configuration.CertificateAuthorityMutualTLS, + TLSMode: configuration.CertificateAuthorityMutualTLS, Certificates: &certification.Certificates{ Certificates: certificates, }, } - _, err := NewTlsConfig(&settings) + _, err := NewTLSConfig(&settings) if err == nil { t.Fatalf(`Expected an error`) } diff --git a/internal/certification/certificates.go b/internal/certification/certificates.go index 53bd843..3ecbc46 100644 --- a/internal/certification/certificates.go +++ b/internal/certification/certificates.go @@ -2,7 +2,8 @@ * Copyright 2023 F5 Inc. All rights reserved. * Use of this source code is governed by the Apache License that can be found in the LICENSE file. * - * Establishes a Watcher for the Kubernetes Secrets that contain the various certificates and keys used to generate a tls.Config object; + * Establishes a Watcher for the Kubernetes Secrets that contain the various certificates + * and keys used to generate a tls.Config object; * exposes the certificates and keys. */ @@ -86,10 +87,7 @@ func (c *Certificates) Initialize() error { c.Certificates = make(map[string]map[string]core.SecretBytes) - informer, err := c.buildInformer() - if err != nil { - return fmt.Errorf(`error occurred building an informer: %w`, err) - } + informer := c.buildInformer() c.informer = informer @@ -116,14 +114,14 @@ func (c *Certificates) Run() error { return nil } -func (c *Certificates) buildInformer() (cache.SharedInformer, error) { +func (c *Certificates) buildInformer() cache.SharedInformer { logrus.Debug("Certificates::buildInformer") options := informers.WithNamespace(SecretsNamespace) factory := informers.NewSharedInformerFactoryWithOptions(c.k8sClient, 0, options) informer := factory.Core().V1().Secrets().Informer() - return informer, nil + return informer } func (c *Certificates) initializeEventHandlers() error { diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go index d9d61ac..89b21bf 100644 --- a/internal/certification/certificates_test.go +++ b/internal/certification/certificates_test.go @@ -21,6 +21,7 @@ const ( ) func TestNewCertificate(t *testing.T) { + t.Parallel() ctx := context.Background() certificates := NewCertificates(ctx, nil) @@ -31,6 +32,7 @@ func TestNewCertificate(t *testing.T) { } func TestCertificates_Initialize(t *testing.T) { + t.Parallel() certificates := NewCertificates(context.Background(), nil) err := certificates.Initialize() @@ -40,6 +42,7 @@ func TestCertificates_Initialize(t *testing.T) { } func TestCertificates_RunWithoutInitialize(t *testing.T) { + t.Parallel() certificates := NewCertificates(context.Background(), nil) err := certificates.Run() @@ -53,6 +56,7 @@ func TestCertificates_RunWithoutInitialize(t *testing.T) { } func TestCertificates_EmptyCertificates(t *testing.T) { + t.Parallel() certificates := NewCertificates(context.Background(), nil) err := certificates.Initialize() @@ -75,6 +79,7 @@ func TestCertificates_EmptyCertificates(t *testing.T) { } func TestCertificates_ExerciseHandlers(t *testing.T) { + t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -86,6 +91,7 @@ func TestCertificates_ExerciseHandlers(t *testing.T) { certificates.CaCertificateSecretKey = CaCertificateSecretKey + //nolint:govet,staticcheck go func() { err := certificates.Run() if err != nil { diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 8b19490..40bf02a 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -15,12 +15,12 @@ import ( "github.com/sirupsen/logrus" ) -// NewHttpClient is a factory method to create a new Http Client with a default configuration. +// NewHTTPClient is a factory method to create a new Http Client with a default configuration. // RoundTripper is a wrapper around the default net/communication Transport to add additional headers, in this case, // the Headers are configured for JSON. -func NewHttpClient(settings *configuration.Settings) (*netHttp.Client, error) { +func NewHTTPClient(settings *configuration.Settings) (*netHttp.Client, error) { headers := NewHeaders() - tlsConfig := NewTlsConfig(settings) + tlsConfig := NewTLSConfig(settings) transport := NewTransport(tlsConfig) roundTripper := NewRoundTripper(headers, transport) @@ -40,13 +40,13 @@ func NewHeaders() []string { } } -// NewTlsConfig is a factory method to create a new basic Tls Config. +// NewTLSConfig is a factory method to create a new basic Tls Config. // More attention should be given to the use of `InsecureSkipVerify: true`, as it is not recommended for production use. -func NewTlsConfig(settings *configuration.Settings) *tls.Config { - tlsConfig, err := authentication.NewTlsConfig(settings) +func NewTLSConfig(settings *configuration.Settings) *tls.Config { + tlsConfig, err := authentication.NewTLSConfig(settings) if err != nil { logrus.Warnf("Failed to create TLS config: %v", err) - return &tls.Config{InsecureSkipVerify: true} + return &tls.Config{InsecureSkipVerify: true} //nolint:gosec } return tlsConfig diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index f95722d..375da3b 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -13,10 +13,14 @@ import ( "k8s.io/client-go/kubernetes/fake" ) -func TestNewHttpClient(t *testing.T) { +func TestNewHTTPClient(t *testing.T) { + t.Parallel() k8sClient := fake.NewSimpleClientset() settings, err := configuration.NewSettings(context.Background(), k8sClient) - client, err := NewHttpClient(settings) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } + client, err := NewHTTPClient(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) @@ -28,6 +32,7 @@ func TestNewHttpClient(t *testing.T) { } func TestNewHeaders(t *testing.T) { + t.Parallel() headers := NewHeaders() if headers == nil { @@ -48,9 +53,10 @@ func TestNewHeaders(t *testing.T) { } func TestNewTransport(t *testing.T) { + t.Parallel() k8sClient := fake.NewSimpleClientset() settings, _ := configuration.NewSettings(context.Background(), k8sClient) - config := NewTlsConfig(settings) + config := NewTLSConfig(settings) transport := NewTransport(config) if transport == nil { diff --git a/internal/communication/roundtripper.go b/internal/communication/roundtripper.go index 3781c62..58de6f0 100644 --- a/internal/communication/roundtripper.go +++ b/internal/communication/roundtripper.go @@ -7,7 +7,6 @@ package communication import ( "net/http" - netHttp "net/http" "strings" ) @@ -18,7 +17,7 @@ type RoundTripper struct { } // NewRoundTripper is a factory method to create a new RoundTripper. -func NewRoundTripper(headers []string, transport *netHttp.Transport) *RoundTripper { +func NewRoundTripper(headers []string, transport *http.Transport) *RoundTripper { return &RoundTripper{ Headers: headers, RoundTripper: transport, diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index 0a549d6..ff6d5c4 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -16,10 +16,11 @@ import ( ) func TestNewRoundTripper(t *testing.T) { + t.Parallel() k8sClient := fake.NewSimpleClientset() settings, _ := configuration.NewSettings(context.Background(), k8sClient) headers := NewHeaders() - transport := NewTransport(NewTlsConfig(settings)) + transport := NewTransport(NewTLSConfig(settings)) roundTripper := NewRoundTripper(headers, transport) if roundTripper == nil { @@ -48,10 +49,14 @@ func TestNewRoundTripper(t *testing.T) { } func TestRoundTripperRoundTrip(t *testing.T) { + t.Parallel() k8sClient := fake.NewSimpleClientset() settings, err := configuration.NewSettings(context.Background(), k8sClient) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } headers := NewHeaders() - transport := NewTransport(NewTlsConfig(settings)) + transport := NewTransport(NewTLSConfig(settings)) roundTripper := NewRoundTripper(headers, transport) request, err := NewRequest("GET", "http://example.com", nil) @@ -70,6 +75,7 @@ func TestRoundTripperRoundTrip(t *testing.T) { if response == nil { t.Fatalf(`response should not be nil`) } + defer response.Body.Close() headerLen := len(response.Header) if headerLen <= 2 { diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index e6bd30b..05c3690 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -46,7 +46,8 @@ const ( // There are two work queues in the application: // 1. nlk-handler queue, used to move messages between the Watcher and the Handler. // 2. nlk-synchronizer queue, used to move message between the Handler and the Synchronizer. -// The queues are NamedDelayingQueue objects that use an ItemExponentialFailureRateLimiter as the underlying rate limiter. +// The queues are NamedDelayingQueue objects that use an ItemExponentialFailureRateLimiter +// as the underlying rate limiter. type WorkQueueSettings struct { // Name is the name of the queue. Name string @@ -110,8 +111,9 @@ type Settings struct { // NginxPlusHosts is a list of Nginx Plus hosts that will be used to update the Border Servers. NginxPlusHosts []string - // TlsMode is the value used to determine which of the five TLS modes will be used to communicate with the Border Servers (see: ../../docs/tls/README.md). - TlsMode TLSMode + // TlsMode is the value used to determine which of the five TLS modes will be used to communicate + // with the Border Servers (see: ../../docs/tls/README.md). + TLSMode TLSMode // Certificates is the object used to retrieve the certificates and keys used to communicate with the Border Servers. Certificates *certification.Certificates @@ -140,7 +142,7 @@ func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings settings := &Settings{ Context: ctx, K8sClient: k8sClient, - TlsMode: NoTLS, + TLSMode: NoTLS, Certificates: nil, Handler: HandlerSettings{ RetryCount: 5, @@ -187,10 +189,12 @@ func (s *Settings) Initialize() error { s.Certificates = certificates - go certificates.Run() + certificates.Run() //nolint:errcheck logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieving nlk-config ConfigMap") - configMap, err := s.K8sClient.CoreV1().ConfigMaps(ConfigMapsNamespace).Get(s.Context, "nlk-config", metav1.GetOptions{}) + configMap, err := s.K8sClient.CoreV1().ConfigMaps(ConfigMapsNamespace).Get( + s.Context, "nlk-config", metav1.GetOptions{}, + ) if err != nil { return err } @@ -198,10 +202,7 @@ func (s *Settings) Initialize() error { s.handleUpdateEvent(nil, configMap) logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieved nlk-config ConfigMap") - informer, err := s.buildInformer() - if err != nil { - return fmt.Errorf(`error occurred building ConfigMap informer: %w`, err) - } + informer := s.buildInformer() s.informer = informer @@ -213,7 +214,7 @@ func (s *Settings) Initialize() error { return nil } -// Run starts the SharedInformer and waits for the Context to be cancelled. +// Run starts the SharedInformer and waits for the Context to be canceled. func (s *Settings) Run() { logrus.Debug("Settings::Run") @@ -224,12 +225,12 @@ func (s *Settings) Run() { <-s.Context.Done() } -func (s *Settings) buildInformer() (cache.SharedInformer, error) { +func (s *Settings) buildInformer() cache.SharedInformer { options := informers.WithNamespace(ConfigMapsNamespace) factory := informers.NewSharedInformerFactoryWithOptions(s.K8sClient, ResyncPeriod, options) informer := factory.Core().V1().ConfigMaps().Informer() - return informer, nil + return informer } func (s *Settings) initializeEventListeners() error { @@ -283,12 +284,15 @@ func (s *Settings) handleUpdateEvent(_ interface{}, newValue interface{}) { logrus.Warnf("Settings::handleUpdateEvent: nginx-hosts key not found in ConfigMap") } - tlsMode, err := validateTlsMode(configMap) + tlsMode, err := validateTLSMode(configMap) if err != nil { // NOTE: the TLSMode defaults to NoTLS on startup, or the last known good value if previously set. - logrus.Errorf("There was an error with the configured TLS Mode. TLS Mode has NOT been changed. The current mode is: '%v'. Error: %v. ", s.TlsMode, err) + logrus.Errorf( + "Error with configured TLS Mode. TLS Mode has NOT been changed. The current mode is: '%v'. Error: %v. ", + s.TLSMode, err, + ) } else { - s.TlsMode = tlsMode + s.TLSMode = tlsMode } caCertificateSecretKey, found := configMap.Data["ca-certificate"] @@ -314,7 +318,7 @@ func (s *Settings) handleUpdateEvent(_ interface{}, newValue interface{}) { logrus.Debugf("Settings::handleUpdateEvent: \n\tHosts: %v,\n\tSettings: %v ", s.NginxPlusHosts, configMap) } -func validateTlsMode(configMap *corev1.ConfigMap) (TLSMode, error) { +func validateTLSMode(configMap *corev1.ConfigMap) (TLSMode, error) { tlsConfigMode, tlsConfigModeFound := configMap.Data["tls-mode"] if !tlsConfigModeFound { return NoTLS, fmt.Errorf(`tls-mode key not found in ConfigMap`) diff --git a/internal/configuration/tlsmodes_test.go b/internal/configuration/tlsmodes_test.go index 62abf96..d849cd9 100644 --- a/internal/configuration/tlsmodes_test.go +++ b/internal/configuration/tlsmodes_test.go @@ -10,6 +10,7 @@ import ( ) func Test_String(t *testing.T) { + t.Parallel() mode := NoTLS.String() if mode != "no-tls" { t.Errorf("Expected TLSModeNoTLS to be 'no-tls', got '%s'", mode) @@ -42,6 +43,7 @@ func Test_String(t *testing.T) { } func Test_TLSModeMap(t *testing.T) { + t.Parallel() mode := TLSModeMap["no-tls"] if mode != NoTLS { t.Errorf("Expected TLSModeMap['no-tls'] to be TLSModeNoTLS, got '%d'", mode) diff --git a/internal/core/event_test.go b/internal/core/event_test.go index 7f7448d..662eb8f 100644 --- a/internal/core/event_test.go +++ b/internal/core/event_test.go @@ -7,6 +7,7 @@ import ( ) func TestNewEvent(t *testing.T) { + t.Parallel() expectedType := Created expectedService := &v1.Service{} expectedPreviousService := &v1.Service{} diff --git a/internal/core/secret_bytes_test.go b/internal/core/secret_bytes_test.go index 8dd8024..5e6bc3f 100644 --- a/internal/core/secret_bytes_test.go +++ b/internal/core/secret_bytes_test.go @@ -12,6 +12,7 @@ import ( ) func TestSecretBytesToString(t *testing.T) { + t.Parallel() sensitive := SecretBytes([]byte("If you can see this we have a problem")) expected := "foo [REDACTED] bar" @@ -22,6 +23,7 @@ func TestSecretBytesToString(t *testing.T) { } func TestSecretBytesToJSON(t *testing.T) { + t.Parallel() sensitive, _ := json.Marshal(SecretBytes([]byte("If you can see this we have a problem"))) expected := `foo "[REDACTED]" bar` result := fmt.Sprintf("foo %v bar", string(sensitive)) diff --git a/internal/core/server_update_event.go b/internal/core/server_update_event.go index f3961ea..0bc8868 100644 --- a/internal/core/server_update_event.go +++ b/internal/core/server_update_event.go @@ -15,7 +15,7 @@ type ServerUpdateEvent struct { ClientType string // Id is the unique identifier for this event. - Id string + ID string // NginxHost is the host name of the NGINX Plus instance that should handle this event. NginxHost string @@ -34,7 +34,12 @@ type ServerUpdateEvent struct { type ServerUpdateEvents = []*ServerUpdateEvent // NewServerUpdateEvent creates a new ServerUpdateEvent. -func NewServerUpdateEvent(eventType EventType, upstreamName string, clientType string, upstreamServers UpstreamServers) *ServerUpdateEvent { +func NewServerUpdateEvent( + eventType EventType, + upstreamName string, + clientType string, + upstreamServers UpstreamServers, +) *ServerUpdateEvent { return &ServerUpdateEvent{ ClientType: clientType, Type: eventType, @@ -43,11 +48,11 @@ func NewServerUpdateEvent(eventType EventType, upstreamName string, clientType s } } -// ServerUpdateEventWithIdAndHost creates a new ServerUpdateEvent with the specified Id and Host. -func ServerUpdateEventWithIdAndHost(event *ServerUpdateEvent, id string, nginxHost string) *ServerUpdateEvent { +// ServerUpdateEventWithIDAndHost creates a new ServerUpdateEvent with the specified Id and Host. +func ServerUpdateEventWithIDAndHost(event *ServerUpdateEvent, id string, nginxHost string) *ServerUpdateEvent { return &ServerUpdateEvent{ ClientType: event.ClientType, - Id: id, + ID: id, NginxHost: nginxHost, Type: event.Type, UpstreamName: event.UpstreamName, diff --git a/internal/core/server_update_event_test.go b/internal/core/server_update_event_test.go index a891e23..9f36002 100644 --- a/internal/core/server_update_event_test.go +++ b/internal/core/server_update_event_test.go @@ -14,32 +14,34 @@ const clientType = "clientType" var emptyUpstreamServers UpstreamServers func TestServerUpdateEventWithIdAndHost(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Created, "upstream", clientType, emptyUpstreamServers) - if event.Id != "" { - t.Errorf("expected empty Id, got %s", event.Id) + if event.ID != "" { + t.Errorf("expected empty ID, got %s", event.ID) } if event.NginxHost != "" { t.Errorf("expected empty NginxHost, got %s", event.NginxHost) } - eventWithIdAndHost := ServerUpdateEventWithIdAndHost(event, "id", "host") + eventWithIDAndHost := ServerUpdateEventWithIDAndHost(event, "id", "host") - if eventWithIdAndHost.Id != "id" { - t.Errorf("expected Id to be 'id', got %s", eventWithIdAndHost.Id) + if eventWithIDAndHost.ID != "id" { + t.Errorf("expected Id to be 'id', got %s", eventWithIDAndHost.ID) } - if eventWithIdAndHost.NginxHost != "host" { - t.Errorf("expected NginxHost to be 'host', got %s", eventWithIdAndHost.NginxHost) + if eventWithIDAndHost.NginxHost != "host" { + t.Errorf("expected NginxHost to be 'host', got %s", eventWithIDAndHost.NginxHost) } - if eventWithIdAndHost.ClientType != clientType { - t.Errorf("expected ClientType to be '%s', got %s", clientType, eventWithIdAndHost.ClientType) + if eventWithIDAndHost.ClientType != clientType { + t.Errorf("expected ClientType to be '%s', got %s", clientType, eventWithIDAndHost.ClientType) } } func TestTypeNameCreated(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Created, "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Created" { @@ -48,6 +50,7 @@ func TestTypeNameCreated(t *testing.T) { } func TestTypeNameUpdated(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Updated, "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Updated" { @@ -56,6 +59,7 @@ func TestTypeNameUpdated(t *testing.T) { } func TestTypeNameDeleted(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(Deleted, "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Deleted" { @@ -64,6 +68,7 @@ func TestTypeNameDeleted(t *testing.T) { } func TestTypeNameUnknown(t *testing.T) { + t.Parallel() event := NewServerUpdateEvent(EventType(100), "upstream", clientType, emptyUpstreamServers) if event.TypeName() != "Unknown" { diff --git a/internal/core/upstream_server.go b/internal/core/upstream_server.go index 7c89b1e..eeb72ac 100644 --- a/internal/core/upstream_server.go +++ b/internal/core/upstream_server.go @@ -5,7 +5,8 @@ package core -// UpstreamServer represents a single upstream server. This is an internal representation used to abstract the definition +// UpstreamServer represents a single upstream server. +// This is an internal representation used to abstract the definition // of an upstream server from any specific client. type UpstreamServer struct { diff --git a/internal/core/upstream_server_test.go b/internal/core/upstream_server_test.go index 7b0eed5..91592cd 100644 --- a/internal/core/upstream_server_test.go +++ b/internal/core/upstream_server_test.go @@ -8,6 +8,7 @@ package core import "testing" func TestNewUpstreamServer(t *testing.T) { + t.Parallel() host := "localhost" us := NewUpstreamServer(host) if us.Host != host { diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 8e89745..5584939 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -47,7 +47,11 @@ type Handler struct { } // NewHandler creates a new event handler -func NewHandler(settings *configuration.Settings, synchronizer synchronization.Interface, eventQueue workqueue.RateLimitingInterface) *Handler { +func NewHandler( + settings *configuration.Settings, + synchronizer synchronization.Interface, + eventQueue workqueue.RateLimitingInterface, +) *Handler { return &Handler{ eventQueue: eventQueue, settings: settings, diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index e5bef3d..b7a4791 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -18,6 +18,7 @@ import ( ) func TestHandler_AddsEventToSynchronizer(t *testing.T) { + t.Parallel() _, _, synchronizer, handler, err := buildHandler() if err != nil { t.Errorf(`should have been no error, %v`, err) @@ -45,7 +46,11 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { } } -func buildHandler() (*configuration.Settings, workqueue.RateLimitingInterface, *mocks.MockSynchronizer, *Handler, error) { +func buildHandler() ( + *configuration.Settings, + workqueue.RateLimitingInterface, + *mocks.MockSynchronizer, *Handler, error, +) { settings, err := configuration.NewSettings(context.Background(), nil) if err != nil { return nil, nil, nil, nil, fmt.Errorf(`should have been no error, %v`, err) diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 74e2d05..a022f2e 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -51,10 +51,7 @@ func (w *Watcher) Initialize() error { logrus.Debug("Watcher::Initialize") var err error - w.informer, err = w.buildInformer() - if err != nil { - return fmt.Errorf(`initialization error: %w`, err) - } + w.informer = w.buildInformer() err = w.initializeEventListeners() if err != nil { @@ -78,7 +75,11 @@ func (w *Watcher) Watch() error { go w.informer.Run(w.settings.Context.Done()) - if !cache.WaitForNamedCacheSync(w.settings.Handler.WorkQueueSettings.Name, w.settings.Context.Done(), w.informer.HasSynced) { + if !cache.WaitForNamedCacheSync( + w.settings.Handler.WorkQueueSettings.Name, + w.settings.Context.Done(), + w.informer.HasSynced, + ) { return fmt.Errorf(`error occurred waiting for the cache to sync`) } @@ -86,7 +87,8 @@ func (w *Watcher) Watch() error { return nil } -// buildEventHandlerForAdd creates a function that is used as an event handler for the informer when Add events are raised. +// buildEventHandlerForAdd creates a function that is used as an event handler +// for the informer when Add events are raised. func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { logrus.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { @@ -102,7 +104,8 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { } } -// buildEventHandlerForDelete creates a function that is used as an event handler for the informer when Delete events are raised. +// buildEventHandlerForDelete creates a function that is used as an event handler +// for the informer when Delete events are raised. func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { logrus.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { @@ -118,7 +121,8 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { } } -// buildEventHandlerForUpdate creates a function that is used as an event handler for the informer when Update events are raised. +// buildEventHandlerForUpdate creates a function that is used as an event handler +// for the informer when Update events are raised. func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { logrus.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { @@ -135,14 +139,16 @@ func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { } // buildInformer creates the informer used to watch for changes to Kubernetes resources. -func (w *Watcher) buildInformer() (cache.SharedIndexInformer, error) { +func (w *Watcher) buildInformer() cache.SharedIndexInformer { logrus.Debug("Watcher::buildInformer") options := informers.WithNamespace(w.settings.Watcher.NginxIngressNamespace) - factory := informers.NewSharedInformerFactoryWithOptions(w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, options) + factory := informers.NewSharedInformerFactoryWithOptions( + w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, options, + ) informer := factory.Core().V1().Services().Informer() - return informer, nil + return informer } // initializeEventListeners initializes the event listeners for the informer. diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 7c820bf..2a6d94b 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -15,6 +15,7 @@ import ( ) func TestWatcher_MustInitialize(t *testing.T) { + t.Parallel() watcher, _ := buildWatcher() if err := watcher.Watch(); err == nil { t.Errorf("Expected error, got %s", err) diff --git a/internal/probation/check_test.go b/internal/probation/check_test.go index 208c9a4..95358e5 100644 --- a/internal/probation/check_test.go +++ b/internal/probation/check_test.go @@ -8,6 +8,7 @@ package probation import "testing" func TestCheck_LiveCheck(t *testing.T) { + t.Parallel() check := LiveCheck{} if !check.Check() { t.Errorf("LiveCheck should return true") @@ -15,6 +16,7 @@ func TestCheck_LiveCheck(t *testing.T) { } func TestCheck_ReadyCheck(t *testing.T) { + t.Parallel() check := ReadyCheck{} if !check.Check() { t.Errorf("ReadyCheck should return true") @@ -22,6 +24,7 @@ func TestCheck_ReadyCheck(t *testing.T) { } func TestCheck_StartupCheck(t *testing.T) { + t.Parallel() check := StartupCheck{} if !check.Check() { t.Errorf("StartupCheck should return true") diff --git a/internal/probation/server.go b/internal/probation/server.go index ff23e33..84b5b67 100644 --- a/internal/probation/server.go +++ b/internal/probation/server.go @@ -8,6 +8,7 @@ package probation import ( "fmt" "net/http" + "time" "github.com/sirupsen/logrus" ) @@ -59,7 +60,7 @@ func (hs *HealthServer) Start() { mux.HandleFunc("/livez", hs.HandleLive) mux.HandleFunc("/readyz", hs.HandleReady) mux.HandleFunc("/startupz", hs.HandleStartup) - hs.httpServer = &http.Server{Addr: address, Handler: mux} + hs.httpServer = &http.Server{Addr: address, Handler: mux, ReadTimeout: 2 * time.Second} go func() { if err := hs.httpServer.ListenAndServe(); err != nil { diff --git a/internal/probation/server_test.go b/internal/probation/server_test.go index 4ea51e7..9c7d37d 100644 --- a/internal/probation/server_test.go +++ b/internal/probation/server_test.go @@ -14,6 +14,7 @@ import ( ) func TestHealthServer_HandleLive(t *testing.T) { + t.Parallel() server := NewHealthServer() writer := mocks.NewMockResponseWriter() server.HandleLive(writer, nil) @@ -24,6 +25,7 @@ func TestHealthServer_HandleLive(t *testing.T) { } func TestHealthServer_HandleReady(t *testing.T) { + t.Parallel() server := NewHealthServer() writer := mocks.NewMockResponseWriter() server.HandleReady(writer, nil) @@ -34,6 +36,7 @@ func TestHealthServer_HandleReady(t *testing.T) { } func TestHealthServer_HandleStartup(t *testing.T) { + t.Parallel() server := NewHealthServer() writer := mocks.NewMockResponseWriter() server.HandleStartup(writer, nil) @@ -44,6 +47,7 @@ func TestHealthServer_HandleStartup(t *testing.T) { } func TestHealthServer_HandleFailCheck(t *testing.T) { + t.Parallel() failCheck := mocks.NewMockCheck(false) server := NewHealthServer() writer := mocks.NewMockResponseWriter() @@ -56,6 +60,7 @@ func TestHealthServer_HandleFailCheck(t *testing.T) { } func TestHealthServer_Start(t *testing.T) { + t.Parallel() server := NewHealthServer() server.Start() @@ -65,6 +70,7 @@ func TestHealthServer_Start(t *testing.T) { if err != nil { t.Error(err) } + defer response.Body.Close() if response.StatusCode != http.StatusOK { t.Errorf("Expected status code %v, got %v", http.StatusAccepted, response.StatusCode) diff --git a/internal/synchronization/rand.go b/internal/synchronization/rand.go index 425b99a..6bf58d1 100644 --- a/internal/synchronization/rand.go +++ b/internal/synchronization/rand.go @@ -6,6 +6,7 @@ package synchronization import ( + // Try using crpyto if needed. "math/rand" "time" ) @@ -24,14 +25,14 @@ func RandomString(n int) string { b := make([]byte, n) for i := range b { // randomly select 1 character from given charset - b[i] = alphaNumeric[rand.Intn(len(alphaNumeric))] + b[i] = alphaNumeric[rand.Intn(len(alphaNumeric))] //nolint:gosec } return string(b) } // RandomMilliseconds returns a random duration between min and max milliseconds func RandomMilliseconds(min, max int) time.Duration { - randomizer := rand.New(rand.NewSource(time.Now().UnixNano())) + randomizer := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec random := randomizer.Intn(max-min) + min return time.Millisecond * time.Duration(random) diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 5fc07f5..7522f3a 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -35,15 +35,19 @@ type Interface interface { } // Synchronizer is responsible for synchronizing the state of the Border Servers. -// Operating against the "nlk-synchronizer", it handles events by creating a Border Client as specified in the -// Service annotation for the Upstream. see application/border_client.go and application/application_constants.go for details. +// Operating against the "nlk-synchronizer", it handles events by creating +// a Border Client as specified in the Service annotation for the Upstream. +// See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { eventQueue workqueue.RateLimitingInterface settings *configuration.Settings } // NewSynchronizer creates a new Synchronizer. -func NewSynchronizer(settings *configuration.Settings, eventQueue workqueue.RateLimitingInterface) (*Synchronizer, error) { +func NewSynchronizer( + settings *configuration.Settings, + eventQueue workqueue.RateLimitingInterface, +) (*Synchronizer, error) { synchronizer := Synchronizer{ eventQueue: eventQueue, settings: settings, @@ -79,7 +83,10 @@ func (s *Synchronizer) AddEvent(event *core.ServerUpdateEvent) { return } - after := RandomMilliseconds(s.settings.Synchronizer.MinMillisecondsJitter, s.settings.Synchronizer.MaxMillisecondsJitter) + after := RandomMilliseconds( + s.settings.Synchronizer.MinMillisecondsJitter, + s.settings.Synchronizer.MaxMillisecondsJitter, + ) s.eventQueue.AddAfter(event, after) } @@ -108,7 +115,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica var err error - httpClient, err := communication.NewHttpClient(s.settings) + httpClient, err := communication.NewHTTPClient(s.settings) if err != nil { return nil, fmt.Errorf(`error creating HTTP client: %v`, err) } @@ -130,7 +137,7 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se for hidx, host := range s.settings.NginxPlusHosts { for eidx, event := range event { id := fmt.Sprintf(`[%d:%d]-[%s]-[%s]-[%s]`, hidx, eidx, RandomString(12), event.UpstreamName, host) - updatedEvent := core.ServerUpdateEventWithIdAndHost(event, id, host) + updatedEvent := core.ServerUpdateEventWithIDAndHost(event, id, host) events = append(events, updatedEvent) } @@ -141,7 +148,7 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se // handleEvent dispatches an event to the proper handler function. func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleEvent: Id: %s`, event.Id) + logrus.Debugf(`Synchronizer::handleEvent: Id: %s`, event.ID) var err error @@ -160,7 +167,9 @@ func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { } if err == nil { - logrus.Infof(`Synchronizer::handleEvent: successfully %s the nginx+ host(s) for Upstream: %s: Id(%s)`, event.TypeName(), event.UpstreamName, event.Id) + logrus.Infof( + `Synchronizer::handleEvent: successfully %s the nginx+ host(s) for Upstream: %s: Id(%s)`, + event.TypeName(), event.UpstreamName, event.ID) } return err @@ -168,7 +177,7 @@ func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { // handleCreatedUpdatedEvent handles events of type Created or Updated. func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleCreatedUpdatedEvent: Id: %s`, serverUpdateEvent.Id) + logrus.Debugf(`Synchronizer::handleCreatedUpdatedEvent: Id: %s`, serverUpdateEvent.ID) var err error @@ -186,7 +195,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU // handleDeletedEvent handles events of type Deleted. func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleDeletedEvent: Id: %s`, serverUpdateEvent.Id) + logrus.Debugf(`Synchronizer::handleDeletedEvent: Id: %s`, serverUpdateEvent.ID) var err error @@ -233,7 +242,7 @@ func (s *Synchronizer) withRetry(err error, event *core.ServerUpdateEvent) { // TODO: Add Telemetry if s.eventQueue.NumRequeues(event) < s.settings.Synchronizer.RetryCount { // TODO: Make this configurable s.eventQueue.AddRateLimited(event) - logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.Id, err) + logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.ID, err) } else { s.eventQueue.Forget(event) logrus.Warnf(`Synchronizer::withRetry: event %#v has been dropped due to too many retries`, event) diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index ef510b8..3634513 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -16,7 +16,11 @@ import ( ) func TestSynchronizer_NewSynchronizer(t *testing.T) { + t.Parallel() settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } rateLimiter := &mocks.MockRateLimiter{} @@ -31,15 +35,19 @@ func TestSynchronizer_NewSynchronizer(t *testing.T) { } func TestSynchronizer_AddEventNoHosts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 event := &core.ServerUpdateEvent{ - Id: "", + ID: "", NginxHost: "", Type: 0, UpstreamName: "", UpstreamServers: nil, } settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } rateLimiter := &mocks.MockRateLimiter{} synchronizer, err := NewSynchronizer(settings, rateLimiter) @@ -61,9 +69,13 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { } func TestSynchronizer_AddEventOneHost(t *testing.T) { + t.Parallel() const expectedEventCount = 1 events := buildEvents(1) settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } settings.NginxPlusHosts = []string{"https://localhost:8080"} rateLimiter := &mocks.MockRateLimiter{} @@ -84,9 +96,13 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { } func TestSynchronizer_AddEventManyHosts(t *testing.T) { + t.Parallel() const expectedEventCount = 1 events := buildEvents(1) settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } settings.NginxPlusHosts = []string{ "https://localhost:8080", "https://localhost:8081", @@ -111,9 +127,13 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { } func TestSynchronizer_AddEventsNoHosts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 events := buildEvents(4) settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } rateLimiter := &mocks.MockRateLimiter{} synchronizer, err := NewSynchronizer(settings, rateLimiter) @@ -135,9 +155,13 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { } func TestSynchronizer_AddEventsOneHost(t *testing.T) { + t.Parallel() const expectedEventCount = 4 events := buildEvents(4) settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } settings.NginxPlusHosts = []string{"https://localhost:8080"} rateLimiter := &mocks.MockRateLimiter{} @@ -158,10 +182,14 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { } func TestSynchronizer_AddEventsManyHosts(t *testing.T) { + t.Parallel() const eventCount = 4 events := buildEvents(eventCount) rateLimiter := &mocks.MockRateLimiter{} settings, err := configuration.NewSettings(context.Background(), nil) + if err != nil { + t.Fatalf(`Unexpected error: %v`, err) + } settings.NginxPlusHosts = []string{ "https://localhost:8080", "https://localhost:8081", @@ -189,7 +217,7 @@ func buildEvents(count int) core.ServerUpdateEvents { events := make(core.ServerUpdateEvents, count) for i := 0; i < count; i++ { events[i] = &core.ServerUpdateEvent{ - Id: fmt.Sprintf("id-%v", i), + ID: fmt.Sprintf("id-%v", i), NginxHost: "https://localhost:8080", Type: 0, UpstreamName: "", diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 98ff368..5dc49ec 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -40,16 +40,18 @@ func filterPorts(ports []v1.ServicePort) []v1.ServicePort { } // buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type -// The NGINX+ Client uses a list of servers for Created and Updated events; the client performs reconciliation between -// the list of servers in the NGINX+ Client call and the list of servers in NGINX+. -// The NGINX+ Client uses a single server for Deleted events; so the list of servers is broken up into individual events. +// The NGINX+ Client uses a list of servers for Created and Updated events. +// The client performs reconciliation between the list of servers in the NGINX+ Client call +// and the list of servers in NGINX+. +// The NGINX+ Client uses a single server for Deleted events; +// so the list of servers is broken up into individual events. func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.ServerUpdateEvents, error) { logrus.Debugf("Translate::buildServerUpdateEvents(ports=%#v)", ports) events := core.ServerUpdateEvents{} for _, port := range ports { ingressName := fixIngressName(port.Name) - upstreamServers, _ := buildUpstreamServers(event.NodeIps, port) + upstreamServers := buildUpstreamServers(event.NodeIps, port) clientType := getClientType(port.Name, event.Service.Annotations) switch event.Type { @@ -61,7 +63,9 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se case core.Deleted: for _, server := range upstreamServers { - events = append(events, core.NewServerUpdateEvent(event.Type, ingressName, clientType, core.UpstreamServers{server})) + events = append(events, core.NewServerUpdateEvent( + event.Type, ingressName, clientType, core.UpstreamServers{server}, + )) } default: @@ -73,16 +77,16 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se return events, nil } -func buildUpstreamServers(nodeIps []string, port v1.ServicePort) (core.UpstreamServers, error) { +func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamServers { var servers core.UpstreamServers - for _, nodeIp := range nodeIps { - host := fmt.Sprintf("%s:%d", nodeIp, port.NodePort) + for _, nodeIP := range nodeIPs { + host := fmt.Sprintf("%s:%d", nodeIP, port.NodePort) server := core.NewUpstreamServer(host) servers = append(servers, server) } - return servers, nil + return servers } // fixIngressName removes the NlkPrefix from the port name @@ -100,5 +104,5 @@ func getClientType(portName string, annotations map[string]string) string { } } - return application.ClientTypeNginxHttp + return application.ClientTypeNginxHTTP } diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index a393f64..b53abcc 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -29,6 +29,7 @@ const ( */ func TestCreatedTranslateNoPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -46,6 +47,7 @@ func TestCreatedTranslateNoPorts(t *testing.T) { } func TestCreatedTranslateNoInterestingPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 1 @@ -65,6 +67,7 @@ func TestCreatedTranslateNoInterestingPorts(t *testing.T) { } func TestCreatedTranslateOneInterestingPort(t *testing.T) { + t.Parallel() const expectedEventCount = 1 const portCount = 1 @@ -86,6 +89,7 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { } func TestCreatedTranslateManyInterestingPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 4 const portCount = 4 @@ -107,6 +111,7 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { } func TestCreatedTranslateManyMixedPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -129,6 +134,7 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { } func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -155,6 +161,7 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { */ func TestUpdatedTranslateNoPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -172,6 +179,7 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { } func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 1 @@ -191,6 +199,7 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { } func TestUpdatedTranslateOneInterestingPort(t *testing.T) { + t.Parallel() const expectedEventCount = 1 const portCount = 1 @@ -212,6 +221,7 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { } func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 4 const portCount = 4 @@ -233,6 +243,7 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { } func TestUpdatedTranslateManyMixedPorts(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -255,6 +266,7 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { } func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -281,6 +293,7 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { */ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -300,6 +313,7 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { } func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 1 @@ -321,6 +335,8 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { } func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 const portCount = 1 @@ -342,6 +358,7 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { } func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 4 @@ -363,6 +380,7 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { } func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 6 const updatablePortCount = 2 @@ -385,6 +403,7 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { } func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -404,6 +423,7 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { } func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 0 const portCount = 1 @@ -425,6 +445,7 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { } func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 1 const portCount = 1 @@ -446,6 +467,7 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { } func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 4 const portCount = 4 @@ -467,6 +489,7 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { } func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { + t.Parallel() const expectedEventCount = 2 const portCount = 6 const updatablePortCount = 2 @@ -489,6 +512,7 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { } func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { + t.Parallel() const expectedEventCount = 0 service := defaultService() @@ -508,6 +532,7 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { } func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { + t.Parallel() const portCount = 1 const updatablePortCount = 0 const expectedEventCount = updatablePortCount * ManyNodes @@ -530,6 +555,7 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { } func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { + t.Parallel() const portCount = 1 const expectedEventCount = portCount * ManyNodes @@ -551,6 +577,7 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { } func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { + t.Parallel() const portCount = 4 const expectedEventCount = portCount * ManyNodes @@ -572,6 +599,7 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { } func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { + t.Parallel() const portCount = 6 const updatablePortCount = 2 const expectedEventCount = updatablePortCount * ManyNodes @@ -650,7 +678,7 @@ func generatePorts(portCount int) []v1.ServicePort { // This is probably A Little Bit of Too Much™, but helps to ensure ordering is not a factor. func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort { - var ports []v1.ServicePort + ports := []v1.ServicePort{} updatable := make([]string, updatableCount) nonupdatable := make([]string, portCount-updatableCount) @@ -663,7 +691,9 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort nonupdatable[j] = "olm-" } - prefixes := append(updatable, nonupdatable...) + var prefixes []string + prefixes = append(prefixes, updatable...) + prefixes = append(prefixes, nonupdatable...) source := rand.NewSource(time.Now().UnixNano()) random := rand.New(source) diff --git a/test/mocks/mock_nginx_plus_client.go b/test/mocks/mock_nginx_plus_client.go index 2c16e12..00b560e 100644 --- a/test/mocks/mock_nginx_plus_client.go +++ b/test/mocks/mock_nginx_plus_client.go @@ -36,7 +36,10 @@ func (m MockNginxClient) DeleteStreamServer(_ string, _ string) error { return nil } -func (m MockNginxClient) UpdateStreamServers(_ string, _ []nginxClient.StreamUpstreamServer) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) { +func (m MockNginxClient) UpdateStreamServers( + _ string, + _ []nginxClient.StreamUpstreamServer, +) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) { m.CalledFunctions["UpdateStreamServers"] = true if m.Error != nil { @@ -56,7 +59,10 @@ func (m MockNginxClient) DeleteHTTPServer(_ string, _ string) error { return nil } -func (m MockNginxClient) UpdateHTTPServers(_ string, _ []nginxClient.UpstreamServer) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) { +func (m MockNginxClient) UpdateHTTPServers( + _ string, + _ []nginxClient.UpstreamServer, +) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) { m.CalledFunctions["UpdateHTTPServers"] = true if m.Error != nil { From ef78913f7ce22974ab6328d6723e4c02fa86fe7e Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 29 Mar 2024 18:26:15 -0700 Subject: [PATCH 005/136] Update go version and deps --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 38f6adb..7a95a39 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.19 +go 1.21 require ( github.com/nginxinc/nginx-plus-go-client v0.10.0 diff --git a/go.sum b/go.sum index 867f71f..7d3d0e5 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -160,7 +161,9 @@ github.com/nginxinc/nginx-plus-go-client v0.10.0/go.mod h1:0v3RsQCvRn/IyrMtW+DK6 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -179,6 +182,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From 2f41f5e1a401feea0b071cbe4a0cba345ca74a29 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 8 Jul 2024 15:47:57 -0700 Subject: [PATCH 006/136] Disable exhaustruct linter for now --- .golangci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index d89f3f5..6cb3587 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,7 +7,6 @@ linters: # Supported linters: https://golangci-lint.run/usage/linters/ enable: - errcheck - - exhaustruct - gosimple - govet - ineffassign From 196b8457381756bf5d64c0da91430c77c3a7044a Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 26 Jun 2024 13:39:35 -0600 Subject: [PATCH 007/136] NLB-4655 NLK will retry a work item to update upstreams indefinitely The primary intent behind this is to keep retrying updates which may be made before the controlplane has registered the existence of a named upstream in the customer's NGINX configuration. --- internal/synchronization/synchronizer.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 7522f3a..9d1a697 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -20,7 +20,6 @@ import ( // Interface defines the interface needed to implement a synchronizer. type Interface interface { - // AddEvents adds a list of events to the queue. AddEvents(events core.ServerUpdateEvents) @@ -240,13 +239,8 @@ func (s *Synchronizer) withRetry(err error, event *core.ServerUpdateEvent) { logrus.Debug("Synchronizer::withRetry") if err != nil { // TODO: Add Telemetry - if s.eventQueue.NumRequeues(event) < s.settings.Synchronizer.RetryCount { // TODO: Make this configurable - s.eventQueue.AddRateLimited(event) - logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.ID, err) - } else { - s.eventQueue.Forget(event) - logrus.Warnf(`Synchronizer::withRetry: event %#v has been dropped due to too many retries`, event) - } + s.eventQueue.AddRateLimited(event) + logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.ID, err) } else { s.eventQueue.Forget(event) } // TODO: Add error logging From 46ec67bc003672a86fc278748402fbabcb6eb5b4 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 23 Jul 2024 08:48:55 -0700 Subject: [PATCH 008/136] Update binary/docker img to nginxaas-operator --- Dockerfile | 6 +++--- scripts/build.sh | 2 +- scripts/docker.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0a12779..d80281c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,6 @@ COPY docker-user /etc/passwd USER 101 COPY --from=base-certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -FROM base as nlk -ENTRYPOINT ["/nlk"] -COPY build/nlk / +FROM base as nginxaas-operator +ENTRYPOINT ["/nginxaas-operator"] +COPY build/nginxaas-operator / diff --git a/scripts/build.sh b/scripts/build.sh index aba9ad5..b5be0ed 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -36,5 +36,5 @@ ldflags=( go build \ -v -tags "release osusergo" \ -ldflags "${ldflags[*]}" \ - -o "${BUILD_DIR}/nlk" \ + -o "${BUILD_DIR}/nginxaas-operator" \ "$pkg_path" diff --git a/scripts/docker.sh b/scripts/docker.sh index 0015f49..4193f55 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -69,7 +69,7 @@ parse_args() { } # MAIN -image="nlk" +image="nginxaas-operator" parse_args "$@" init_ci_vars From ac9100a695713b7eaa5dc5564511766310da788d Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 31 Jul 2024 13:57:22 -0700 Subject: [PATCH 009/136] NLB-5341: Disable dep caching Disabling caching temporarily to figure out the issue and we can re-enable it after that. --- .gitlab-ci.yml | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08dfa9a..da34f53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,22 +30,23 @@ workflow: .import-devops-core-services: &import-devops-core-services | source ${CI_PROJECT_DIR}/.devops.sh -.go-cache: - variables: - GOPATH: $CI_PROJECT_DIR/.go - cache: - key: - files: - - go.mod - - go.sum - paths: - - .go/pkg/mod/ +# Disabld due to: NLB-5341 +# .go-cache: +# variables: +# GOPATH: $CI_PROJECT_DIR/.go +# cache: +# key: +# files: +# - go.mod +# - go.sum +# paths: +# - .go/pkg/mod/ -.go-cache-readonly: - extends: - - .go-cache - cache: - policy: pull +# .go-cache-readonly: +# extends: +# - .go-cache +# cache: +# policy: pull .golang-private: &golang-private - | @@ -62,7 +63,7 @@ lint + unit-test + build: image: $DEVTOOLS_IMG extends: - .devops-docker-cicd-large - - .go-cache + # - .go-cache script: - *golang-private - | @@ -93,7 +94,7 @@ unit-test-data-race: GO_DATA_RACE: "true" extends: - .default-runner-large - - .go-cache-readonly + # - .go-cache-readonly script: - *golang-private - time make test @@ -113,7 +114,7 @@ whitesource-scan: stage: lint+test+build extends: - .default-runner - - .go-cache + # - .go-cache - .whitesource-template-go variables: NESTED: "true" From 1f83f55e4d83461c89c90fdb225d64db817759b9 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 7 Aug 2024 13:44:57 -0700 Subject: [PATCH 010/136] Run the informer in go routine NOTE: This commit was accidentally missed out in the first iteration of this fork and is present in upstream: https://github.com/nginxinc/nginx-loadbalancer-kubernetes/blob/main/internal/configuration/settings.go#L189 The code needs to move forward to start the watchers and health server (side note: we should also think about the ordering of these at some point) and not running the infomer in a go routine prevents the program from further execution until the context is canceled. In the current iteration on main, the controller is stuck on the informer, and then k8s kills the service and restarts it since the health server is not up. --- internal/configuration/settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index 05c3690..af7c2d8 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -189,7 +189,7 @@ func (s *Settings) Initialize() error { s.Certificates = certificates - certificates.Run() //nolint:errcheck + go certificates.Run() //nolint:errcheck logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieving nlk-config ConfigMap") configMap, err := s.K8sClient.CoreV1().ConfigMaps(ConfigMapsNamespace).Get( From 8512c6a1623184b3d4dbaed404685ed717c69de0 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 29 Jul 2024 16:22:01 -0600 Subject: [PATCH 011/136] NLB-4617 Watcher filters events by annotation on ingress service name The user specifies the ingress service whose events the application should watch through setting the "service-annotation-match" annotation on the application's config map. Only events with a matching service annotation will be passed by the watcher to the handlers. The informer now listens to events from all namespaces. This frees the end user from the restriction of only using the nginx ingress controller. --- internal/configuration/settings.go | 27 +++++++++++++++------- internal/observation/watcher.go | 37 ++++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index af7c2d8..857bd43 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -40,6 +40,14 @@ const ( // The value of the annotation determines which BorderServer implementation will be used. // See the documentation in the `application/application_constants.go` file for details. PortAnnotationPrefix = "nginxinc.io" + + // ServiceAnnotationMatchKey is the key name of the annotation in the application's config map + // that identifies the ingress service whose events will be monitored. + ServiceAnnotationMatchKey = "service-annotation-match" + + // DefaultServiceAnnotation is the default name of the ingress service whose events will be + // monitored. + DefaultServiceAnnotation = "nginxaas" ) // WorkQueueSettings contains the configuration values needed by the Work Queues. @@ -62,7 +70,6 @@ type WorkQueueSettings struct { // HandlerSettings contains the configuration values needed by the Handler. type HandlerSettings struct { - // RetryCount is the number of times the Handler will attempt to process a message before giving up. RetryCount int @@ -75,9 +82,8 @@ type HandlerSettings struct { // WatcherSettings contains the configuration values needed by the Watcher. type WatcherSettings struct { - - // NginxIngressNamespace is the namespace used to filter Services in the Watcher. - NginxIngressNamespace string + // ServiceAnnotation is the annotation of the ingress service whose events the watcher should monitor. + ServiceAnnotation string // ResyncPeriod is the value used to set the resync period for the underlying SharedInformer. ResyncPeriod time.Duration @@ -85,7 +91,6 @@ type WatcherSettings struct { // SynchronizerSettings contains the configuration values needed by the Synchronizer. type SynchronizerSettings struct { - // MaxMillisecondsJitter is the maximum number of milliseconds that will be applied when adding an event to the queue. MaxMillisecondsJitter int @@ -104,7 +109,6 @@ type SynchronizerSettings struct { // Settings contains the configuration values needed by the application. type Settings struct { - // Context is the context used to control the application. Context context.Context @@ -165,8 +169,8 @@ func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings }, }, Watcher: WatcherSettings{ - NginxIngressNamespace: "nginx-ingress", - ResyncPeriod: 0, + ResyncPeriod: 0, + ServiceAnnotation: DefaultServiceAnnotation, }, } @@ -313,6 +317,13 @@ func (s *Settings) handleUpdateEvent(_ interface{}, newValue interface{}) { logrus.Warnf("Settings::handleUpdateEvent: client-certificate key not found in ConfigMap") } + if serviceAnnotation, found := configMap.Data[ServiceAnnotationMatchKey]; found { + s.Watcher.ServiceAnnotation = serviceAnnotation + } else { + s.Watcher.ServiceAnnotation = DefaultServiceAnnotation + } + logrus.Debugf("Settings::handleUpdateEvent: %s: %s", ServiceAnnotationMatchKey, s.Watcher.ServiceAnnotation) + setLogLevel(configMap.Data["log-level"]) logrus.Debugf("Settings::handleUpdateEvent: \n\tHosts: %v,\n\tSettings: %v ", s.NginxPlusHosts, configMap) diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index a022f2e..a07ef12 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -24,7 +24,6 @@ import ( // Particularly, Services in the namespace defined in the WatcherSettings::NginxIngressNamespace setting. // When a change is detected, an Event is generated and added to the Handler's queue. type Watcher struct { - // eventHandlerRegistration is used to track the event handlers eventHandlerRegistration interface{} @@ -87,17 +86,32 @@ func (w *Watcher) Watch() error { return nil } +// isDesiredService returns whether the user has configured the given service for watching. +func (w *Watcher) isDesiredService(service *v1.Service) bool { + annotation, ok := service.Annotations["nginx.com/nginxaas"] + if !ok { + return false + } + + return annotation == w.settings.Watcher.ServiceAnnotation +} + // buildEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { logrus.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { + service := obj.(*v1.Service) + if !w.isDesiredService(service) { + return + } + nodeIps, err := w.retrieveNodeIps() if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return } - service := obj.(*v1.Service) + var previousService *v1.Service e := core.NewEvent(core.Created, service, previousService, nodeIps) w.handler.AddRateLimitedEvent(&e) @@ -109,12 +123,17 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { logrus.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { + service := obj.(*v1.Service) + if !w.isDesiredService(service) { + return + } + nodeIps, err := w.retrieveNodeIps() if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return } - service := obj.(*v1.Service) + var previousService *v1.Service e := core.NewEvent(core.Deleted, service, previousService, nodeIps) w.handler.AddRateLimitedEvent(&e) @@ -126,12 +145,18 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { logrus.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { + // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API + service := updated.(*v1.Service) + if !w.isDesiredService(service) { + return + } + nodeIps, err := w.retrieveNodeIps() if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return } - service := updated.(*v1.Service) + previousService := previous.(*v1.Service) e := core.NewEvent(core.Updated, service, previousService, nodeIps) w.handler.AddRateLimitedEvent(&e) @@ -142,9 +167,8 @@ func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { func (w *Watcher) buildInformer() cache.SharedIndexInformer { logrus.Debug("Watcher::buildInformer") - options := informers.WithNamespace(w.settings.Watcher.NginxIngressNamespace) factory := informers.NewSharedInformerFactoryWithOptions( - w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, options, + w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, ) informer := factory.Core().V1().Services().Informer() @@ -185,7 +209,6 @@ func (w *Watcher) retrieveNodeIps() ([]string, error) { } for _, node := range nodes.Items { - // this is kind of a broad assumption, should probably make this a configurable option if w.notMasterNode(node) { for _, address := range node.Status.Addresses { From 7446d640869bca61421b547bb7317f04a90720a9 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 26 Jul 2024 09:25:37 -0600 Subject: [PATCH 012/136] NLB-4823 Translator assumes that port names provide context and upstream name The port name should now be formatted like this: "http-tea", where the first part of the string is the context type (either "http" or "stream") and the second part of the string after the hyphen is the name of the upstream. --- internal/observation/handler_test.go | 2 +- internal/translation/translator.go | 55 +++++++++---------------- internal/translation/translator_test.go | 7 ++-- 3 files changed, 25 insertions(+), 39 deletions(-) diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index b7a4791..72b6d8f 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -30,7 +30,7 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ { - Name: "nlk-back", + Name: "http-back", }, }, }, diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 5dc49ec..9dad7f8 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -9,8 +9,6 @@ import ( "fmt" "strings" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" @@ -21,22 +19,7 @@ import ( func Translate(event *core.Event) (core.ServerUpdateEvents, error) { logrus.Debug("Translate::Translate") - portsOfInterest := filterPorts(event.Service.Spec.Ports) - - return buildServerUpdateEvents(portsOfInterest, event) -} - -// filterPorts returns a list of ports that have the NlkPrefix in the port name. -func filterPorts(ports []v1.ServicePort) []v1.ServicePort { - var portsOfInterest []v1.ServicePort - - for _, port := range ports { - if strings.HasPrefix(port.Name, configuration.NlkPrefix) { - portsOfInterest = append(portsOfInterest, port) - } - } - - return portsOfInterest + return buildServerUpdateEvents(event.Service.Spec.Ports, event) } // buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type @@ -50,21 +33,25 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se events := core.ServerUpdateEvents{} for _, port := range ports { - ingressName := fixIngressName(port.Name) + context, upstreamName, err := getContextAndUpstreamName(port) + if err != nil { + logrus.Info(err) + continue + } + upstreamServers := buildUpstreamServers(event.NodeIps, port) - clientType := getClientType(port.Name, event.Service.Annotations) switch event.Type { case core.Created: fallthrough case core.Updated: - events = append(events, core.NewServerUpdateEvent(event.Type, ingressName, clientType, upstreamServers)) + events = append(events, core.NewServerUpdateEvent(event.Type, upstreamName, context, upstreamServers)) case core.Deleted: for _, server := range upstreamServers { events = append(events, core.NewServerUpdateEvent( - event.Type, ingressName, clientType, core.UpstreamServers{server}, + event.Type, upstreamName, context, core.UpstreamServers{server}, )) } @@ -89,20 +76,18 @@ func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamSe return servers } -// fixIngressName removes the NlkPrefix from the port name -func fixIngressName(name string) string { - return name[4:] -} +// getContextAndUpstreamName returns the nginx context being supplied by the port (either "http" or "stream") +// and the upstream name. +func getContextAndUpstreamName(port v1.ServicePort) (clientType string, appName string, err error) { + parts := strings.Split(port.Name, "-") + if len(parts) != 2 { + return clientType, appName, + fmt.Errorf("ignoring port %s because it is not in the format [http|stream]-{upstreamName}", port.Name) + } -// getClientType returns the client type for the port, defaults to ClientTypeNginxHttp if no Annotation is found. -func getClientType(portName string, annotations map[string]string) string { - key := fmt.Sprintf("%s/%s", configuration.PortAnnotationPrefix, portName) - logrus.Infof("getClientType: key=%s", key) - if annotations != nil { - if clientType, ok := annotations[key]; ok { - return clientType - } + if parts[0] != "http" && parts[0] != "stream" { + return clientType, appName, fmt.Errorf("port name %s does not include \"http\" or \"stream\" context", port.Name) } - return application.ClientTypeNginxHTTP + return parts[0], parts[1], nil } diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index b53abcc..5b508c3 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" ) @@ -682,9 +681,11 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort updatable := make([]string, updatableCount) nonupdatable := make([]string, portCount-updatableCount) + contexts := []string{"http-", "stream-"} for i := range updatable { - updatable[i] = configuration.NlkPrefix + randomIndex := int(rand.Float32() * 2.0) + updatable[i] = contexts[randomIndex] } for j := range nonupdatable { @@ -701,7 +702,7 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort for i, prefix := range prefixes { ports = append(ports, v1.ServicePort{ - Name: fmt.Sprintf("%sport-%d", prefix, i), + Name: fmt.Sprintf("%supstream%d", prefix, i), }) } From 6f3046040d8bb248871150457832ad2735a33e6f Mon Sep 17 00:00:00 2001 From: sarna Date: Sat, 17 Aug 2024 16:27:27 -0700 Subject: [PATCH 013/136] NLB-5282: Allow images to be pushed to Dockerhub We need to be able to publish the operator to dockerhub in order to be publicly available for customers. Following what we have in ARP as a release strategy where a release tag action would publish the image to dockerhub. --- .gitlab-ci.yml | 10 ++++++++++ scripts/release.sh | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100755 scripts/release.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da34f53..d59d1bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -177,3 +177,13 @@ tag: - if: '$CI_PIPELINE_SOURCE == "schedule"' when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + +dockerhub-push: + stage: release + image: $DEVTOOLS_IMG + extends: + - .devops-docker-cicd + script: + - ./scripts/release.sh + rules: + - if: '$CI_COMMIT_TAG =~ /^release-[\d]+\.[\d]+\.[\d]+/' diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..7c85792 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [[ "${CI}" != "true" ]]; then + echo "This script is meant to be run in the CI." + exit 1 +fi + +pttn="^release-[0-9]+\.[0-9]+\.[0-9]+" +if ! [[ "${CI_COMMIT_TAG}" =~ $pttn ]]; then + echo "CI_COMMIT_TAG needs to be set to valid semver format." + exit 1 +fi + +ROOT_DIR=$(git rev-parse --show-toplevel) +source ${ROOT_DIR}/.devops.sh + +DOCKERHUB_USERNAME=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".username") +if [[ -z "${DOCKERHUB_USERNAME}" ]]; then + echo "DOCKERHUB_USERNAME needs to be set." + exit 1 +fi + +DOCKERHUB_PASSWORD=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".password") +if [[ -z "${DOCKERHUB_PASSWORD}" ]]; then + echo "DOCKERHUB_PASSWORD needs to be set." + exit 1 +fi + +SRC_REGISTRY="${DEVOPS_DOCKER_URL}" +SRC_PATH="nginx-azure-lb/nginxaas-operator/nginxaas-operator" +SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") +SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" + +DST_REGISTRY="docker.io" +DST_PATH="nginx/nginxaas-operator" +DST_TAG="${CI_COMMIT_TAG}" +DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" + +devops.docker.login +docker pull "${SRC_IMG}" +docker tag "${SRC_IMG}" "${DST_IMG}" + +# Login to Dockerhub and push release image to it. +docker login --username "${DOCKERHUB_USERNAME}" --password "${DOCKERHUB_PASSWORD}" "${DST_REGISTRY}" +docker push "${DST_IMG}" From c1861777a1f23030ca35dd184df75919dfc0c21d Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 31 Jul 2024 10:48:55 -0700 Subject: [PATCH 014/136] Remove unneeded file --- .tool-versions | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .tool-versions diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 09548d5..0000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -golang 1.19.13 From e0e45d830e36fdf5e14a46e511d5c7a493202a18 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 19 Aug 2024 16:21:40 -0700 Subject: [PATCH 015/136] Capture code coverage go test does not have a good way to capture unit-test coverage as part of test runs. This commit captures the error code of the unit test run, runs the coverage generation and then exits based on the test status. --- .gitlab-ci.yml | 2 +- scripts/test.sh | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d59d1bb..fdb3481 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,7 +104,7 @@ unit-test-data-race: always paths: - results - expire_in: 3 hours + expire_in: 1 day reports: junit: results/report.xml rules: diff --git a/scripts/test.sh b/scripts/test.sh index 107cee4..00ca02d 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -10,7 +10,12 @@ fi outfile="${RESULTS_DIR}/coverage.out" mkdir -p "$RESULTS_DIR" go_flags+=("-cover" -coverprofile="$outfile") + +set +e gotestsum --junitfile "${RESULTS_DIR}/report.xml" --format pkgname -- "${go_flags[@]}" ./... +rc=$? +set -e echo "Total code coverage:" go tool cover -func="$outfile" | grep 'total:' | tee "${RESULTS_DIR}/anybadge.out" go tool cover -html="$outfile" -o "${RESULTS_DIR}/coverage.html" +exit $rc From ea11e79fea3a4d9d5b4d8913266c99466c9c68b2 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 19 Aug 2024 18:29:29 -0700 Subject: [PATCH 016/136] Run coverage in a separate job We want to avoid running tests several times on main as it increases the time it takes to publish an image on regular merges. Instead, we can run these tests on a schedule on main and capture code coverage from it. --- .gitlab-ci.yml | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdb3481..92e82f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,6 +58,25 @@ workflow: go env -w GOPRIVATE="gitlab.com/f5" go env +.unit-test-common: + stage: lint+test+build + image: $DEVTOOLS_IMG + extends: + - .default-runner-large + # - .go-cache-readonly + script: + - *golang-private + - time make test + coverage: '/^total:\s+\(statements\)\s+(\d+\.\d+\%)$/' + artifacts: + when: + always + paths: + - results + expire_in: 1 day + reports: + junit: results/report.xml + lint + unit-test + build: stage: lint+test+build image: $DEVTOOLS_IMG @@ -88,28 +107,19 @@ lint + unit-test + build: - if: '$CI_COMMIT_BRANCH || $CI_MERGE_REQUEST_ID' unit-test-data-race: - stage: lint+test+build - image: $DEVTOOLS_IMG variables: GO_DATA_RACE: "true" extends: - - .default-runner-large - # - .go-cache-readonly - script: - - *golang-private - - time make test - coverage: '/^total:\s+\(statements\)\s+(\d+\.\d+\%)$/' - artifacts: - when: - always - paths: - - results - expire_in: 1 day - reports: - junit: results/report.xml + - .unit-test-common rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $GO_DATA_RACE == "true"' +coverage: + extends: + - .unit-test-common + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule"' + whitesource-scan: stage: lint+test+build extends: From bd184eb18f271902aebe45f930820398c84c813c Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 22 Aug 2024 13:15:39 -0600 Subject: [PATCH 017/136] NLB-5360 Upgraded nginx-plus client to v 1.2.2 --- go.mod | 6 ++++-- go.sum | 4 ++-- internal/synchronization/synchronizer.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 7a95a39..118de5f 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.21 +go 1.21.2 + +toolchain go1.21.4 require ( - github.com/nginxinc/nginx-plus-go-client v0.10.0 + github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/sirupsen/logrus v1.9.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 diff --git a/go.sum b/go.sum index 7d3d0e5..46a175a 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginxinc/nginx-plus-go-client v0.10.0 h1:3zsMMkPvRDo8D7ZSprXtbAEW/SDmezZWzxdyS+6oAlc= -github.com/nginxinc/nginx-plus-go-client v0.10.0/go.mod h1:0v3RsQCvRn/IyrMtW+DK6CNkz+PxEsXDJPjQ3yUMBF0= +github.com/nginxinc/nginx-plus-go-client v1.2.2 h1:sl7HqNDDZq2EVu0eQQVoZ6PKYGa4h2dB/Qr5Ib0YKGw= +github.com/nginxinc/nginx-plus-go-client v1.2.2/go.mod h1:n8OFLzrJulJ2fur28Cwa1Qp5DZNS2VicLV+Adt30LQ4= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 9d1a697..5280726 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -119,7 +119,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica return nil, fmt.Errorf(`error creating HTTP client: %v`, err) } - ngxClient, err := nginxClient.NewNginxClient(httpClient, event.NginxHost) + ngxClient, err := nginxClient.NewNginxClient(event.NginxHost, nginxClient.WithHTTPClient(httpClient)) if err != nil { return nil, fmt.Errorf(`error creating Nginx Plus client: %v`, err) } From e613e41693d51cf5b38d83dd8210eea5fd2316fc Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 31 Jul 2024 15:58:35 -0600 Subject: [PATCH 018/136] NLB-5065 Operator adds API Key to header --- internal/communication/factory.go | 13 ++++++--- internal/communication/factory_test.go | 29 +++++++++++++++++++-- internal/communication/roundtripper_test.go | 16 +++++++----- internal/configuration/settings.go | 9 +++++++ 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 40bf02a..64ffe68 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -7,6 +7,7 @@ package communication import ( "crypto/tls" + "fmt" netHttp "net/http" "time" @@ -19,7 +20,7 @@ import ( // RoundTripper is a wrapper around the default net/communication Transport to add additional headers, in this case, // the Headers are configured for JSON. func NewHTTPClient(settings *configuration.Settings) (*netHttp.Client, error) { - headers := NewHeaders() + headers := NewHeaders(settings.APIKey) tlsConfig := NewTLSConfig(settings) transport := NewTransport(tlsConfig) roundTripper := NewRoundTripper(headers, transport) @@ -33,11 +34,17 @@ func NewHTTPClient(settings *configuration.Settings) (*netHttp.Client, error) { } // NewHeaders is a factory method to create a new basic Http Headers slice. -func NewHeaders() []string { - return []string{ +func NewHeaders(apiKey string) []string { + headers := []string{ "Content-Type: application/json", "Accept: application/json", } + + if apiKey != "" { + headers = append(headers, fmt.Sprintf("Authorization: ApiKey %s", apiKey)) + } + + return headers } // NewTLSConfig is a factory method to create a new basic Tls Config. diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index 375da3b..398bb6c 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -21,7 +21,6 @@ func TestNewHTTPClient(t *testing.T) { t.Fatalf(`Unexpected error: %v`, err) } client, err := NewHTTPClient(settings) - if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -31,9 +30,35 @@ func TestNewHTTPClient(t *testing.T) { } } +//nolint:goconst func TestNewHeaders(t *testing.T) { t.Parallel() - headers := NewHeaders() + headers := NewHeaders("fakeKey") + + if headers == nil { + t.Fatalf(`headers should not be nil`) + } + + if len(headers) != 3 { + t.Fatalf(`headers should have 3 elements`) + } + + if headers[0] != "Content-Type: application/json" { + t.Fatalf(`headers[0] should be "Content-Type: application/json"`) + } + + if headers[1] != "Accept: application/json" { + t.Fatalf(`headers[1] should be "Accept: application/json"`) + } + + if headers[2] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`headers[2] should be "Accept: Authorization: ApiKey fakeKey"`) + } +} + +func TestNewHeadersWithNoAPIKey(t *testing.T) { + t.Parallel() + headers := NewHeaders("") if headers == nil { t.Fatalf(`headers should not be nil`) diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index ff6d5c4..abee455 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -19,7 +19,7 @@ func TestNewRoundTripper(t *testing.T) { t.Parallel() k8sClient := fake.NewSimpleClientset() settings, _ := configuration.NewSettings(context.Background(), k8sClient) - headers := NewHeaders() + headers := NewHeaders("fakeKey") transport := NewTransport(NewTLSConfig(settings)) roundTripper := NewRoundTripper(headers, transport) @@ -31,8 +31,8 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers should not be nil`) } - if len(roundTripper.Headers) != 2 { - t.Fatalf(`roundTripper.Headers should have 2 elements`) + if len(roundTripper.Headers) != 3 { + t.Fatalf(`roundTripper.Headers should have 3 elements`) } if roundTripper.Headers[0] != "Content-Type: application/json" { @@ -43,6 +43,10 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers[1] should be "Accept: application/json"`) } + if roundTripper.Headers[2] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`headers[2] should be "Accept: Authorization: ApiKey fakeKey"`) + } + if roundTripper.RoundTripper == nil { t.Fatalf(`roundTripper.RoundTripper should not be nil`) } @@ -55,7 +59,7 @@ func TestRoundTripperRoundTrip(t *testing.T) { if err != nil { t.Fatalf(`Unexpected error: %v`, err) } - headers := NewHeaders() + headers := NewHeaders("fakeKey") transport := NewTransport(NewTLSConfig(settings)) roundTripper := NewRoundTripper(headers, transport) @@ -78,8 +82,8 @@ func TestRoundTripperRoundTrip(t *testing.T) { defer response.Body.Close() headerLen := len(response.Header) - if headerLen <= 2 { - t.Fatalf(`response.Header should have at least 2 elements, found %d`, headerLen) + if headerLen <= 3 { + t.Fatalf(`response.Header should have at least 3 elements, found %d`, headerLen) } } diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index 857bd43..400b311 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -7,7 +7,9 @@ package configuration import ( "context" + "encoding/base64" "fmt" + "os" "strings" "time" @@ -119,6 +121,9 @@ type Settings struct { // with the Border Servers (see: ../../docs/tls/README.md). TLSMode TLSMode + // APIKey is the api key used to authenticate with the dataplane API. + APIKey string + // Certificates is the object used to retrieve the certificates and keys used to communicate with the Border Servers. Certificates *certification.Certificates @@ -143,10 +148,14 @@ type Settings struct { // NewSettings creates a new Settings object with default values. func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings, error) { + // get base64 encoded version of raw api key set by user + apiKey := base64.StdEncoding.EncodeToString([]byte(os.Getenv("NGINXAAS_DATAPLANE_API_KEY"))) + settings := &Settings{ Context: ctx, K8sClient: k8sClient, TLSMode: NoTLS, + APIKey: apiKey, Certificates: nil, Handler: HandlerSettings{ RetryCount: 5, From 713b26c2d7295cffaee20b56816f4eb0445ccf54 Mon Sep 17 00:00:00 2001 From: sarna Date: Sun, 1 Sep 2024 19:57:16 -0700 Subject: [PATCH 019/136] Publish helm charts for development Helm will be used as part of the user story to deploy the operator but it is also a good tool to deploy the operator while developing it. This commit adds the ability to publish helm charts: - to the dev registry for local iteration. - to the regular devops registry for CI iteration and testing. This will also help us test the helm chart itself. --- .gitignore | 2 ++ .gitlab-ci.yml | 1 + Makefile | 4 ++++ scripts/publish-helm.sh | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+) create mode 100755 scripts/publish-helm.sh diff --git a/.gitignore b/.gitignore index 4e1c0fd..396ec9f 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,5 @@ results .go/pkg/mod .go-build + +nginx-loadbalancer-kubernetes-* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92e82f6..610889b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,6 +92,7 @@ lint + unit-test + build: time make test fi time make publish + time make publish-helm coverage: '/^total:\s+\(statements\)\s+(\d+\.\d+\%)$/' artifacts: when: diff --git a/Makefile b/Makefile index 4d404ac..2795d83 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,10 @@ build-linux: build-linux-docker: @./scripts/docker.sh build +publish-helm: + @scripts/docker-login.sh + @scripts/publish-helm.sh + publish: build-linux build-linux-docker @scripts/docker-login.sh @./scripts/docker.sh publish diff --git a/scripts/publish-helm.sh b/scripts/publish-helm.sh new file mode 100755 index 0000000..45e94aa --- /dev/null +++ b/scripts/publish-helm.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eo pipefail + +ROOT_DIR=$(git rev-parse --show-toplevel) + +publish_helm() { + pkg="nginx-loadbalancer-kubernetes-${VERSION}.tgz" + helm package --version "${VERSION}" --app-version "${VERSION}" charts/nlk + helm push "${pkg}" "${repo}" +} + +init_ci_vars() { + if [ -z "$CI_PROJECT_NAME" ]; then + CI_PROJECT_NAME=$(basename "$ROOT_DIR") + fi + if [ -z "$CI_COMMIT_REF_SLUG" ]; then + CI_COMMIT_REF_SLUG=$( + git rev-parse --abbrev-ref HEAD | tr "[:upper:]" "[:lower:]" \ + | LANG=en_US.utf8 sed -E -e 's/[^a-zA-Z0-9]/-/g' -e 's/^-+|-+$$//g' \ + | cut -c 1-63 + ) + fi +} + +# MAIN +init_ci_vars + +# shellcheck source=/dev/null +source "${ROOT_DIR}/.devops.sh" +if [ "$CI" != "true" ]; then + devops.backend.docker.set "azure.container-registry-dev" +fi +repo="oci://${DEVOPS_DOCKER_URL}/nginx-azure-lb/${CI_PROJECT_NAME}/charts/${CI_COMMIT_REF_SLUG}" +# shellcheck source=/dev/null +# shellcheck disable=SC2153 +version=$(source "${ROOT_DIR}/version";echo "$VERSION") + +publish_helm From 294986848f5dd100f21cc01c2558415206a6f2ac Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 22 Aug 2024 17:44:57 -0700 Subject: [PATCH 020/136] Update Chart version to be 0.1.0 0.0.1 is okay but it's not really a bug fix. --- charts/nlk/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index c11d885..e82eaab 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -16,4 +16,4 @@ maintainers: - name: "@abdennour" type: application -version: 0.0.1 +version: 0.1.0 From 608f0081b4df04f3c3ff9a244b4e2adf320a50f6 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 26 Aug 2024 12:57:00 -0700 Subject: [PATCH 021/136] Update release script to handle dual artifacts We should keep a single release script that will publish docker images and helm charts for the official release. This commit just updates the current release script to handle both artifact types. Helm logic will follow. --- .gitlab-ci.yml | 2 +- scripts/release.sh | 113 ++++++++++++++++++++++++++++----------------- 2 files changed, 71 insertions(+), 44 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 610889b..d4c593b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -195,6 +195,6 @@ dockerhub-push: extends: - .devops-docker-cicd script: - - ./scripts/release.sh + - ./scripts/release.sh docker-image rules: - if: '$CI_COMMIT_TAG =~ /^release-[\d]+\.[\d]+\.[\d]+/' diff --git a/scripts/release.sh b/scripts/release.sh index 7c85792..48b18a8 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -2,46 +2,73 @@ set -eo pipefail -if [[ "${CI}" != "true" ]]; then - echo "This script is meant to be run in the CI." - exit 1 -fi - -pttn="^release-[0-9]+\.[0-9]+\.[0-9]+" -if ! [[ "${CI_COMMIT_TAG}" =~ $pttn ]]; then - echo "CI_COMMIT_TAG needs to be set to valid semver format." - exit 1 -fi - -ROOT_DIR=$(git rev-parse --show-toplevel) -source ${ROOT_DIR}/.devops.sh - -DOCKERHUB_USERNAME=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".username") -if [[ -z "${DOCKERHUB_USERNAME}" ]]; then - echo "DOCKERHUB_USERNAME needs to be set." - exit 1 -fi - -DOCKERHUB_PASSWORD=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".password") -if [[ -z "${DOCKERHUB_PASSWORD}" ]]; then - echo "DOCKERHUB_PASSWORD needs to be set." - exit 1 -fi - -SRC_REGISTRY="${DEVOPS_DOCKER_URL}" -SRC_PATH="nginx-azure-lb/nginxaas-operator/nginxaas-operator" -SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") -SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" - -DST_REGISTRY="docker.io" -DST_PATH="nginx/nginxaas-operator" -DST_TAG="${CI_COMMIT_TAG}" -DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" - -devops.docker.login -docker pull "${SRC_IMG}" -docker tag "${SRC_IMG}" "${DST_IMG}" - -# Login to Dockerhub and push release image to it. -docker login --username "${DOCKERHUB_USERNAME}" --password "${DOCKERHUB_PASSWORD}" "${DST_REGISTRY}" -docker push "${DST_IMG}" +docker-image() { + DOCKERHUB_USERNAME=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".username") + if [[ -z "${DOCKERHUB_USERNAME}" ]]; then + echo "DOCKERHUB_USERNAME needs to be set." + exit 1 + fi + + DOCKERHUB_PASSWORD=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".password") + if [[ -z "${DOCKERHUB_PASSWORD}" ]]; then + echo "DOCKERHUB_PASSWORD needs to be set." + exit 1 + fi + + SRC_REGISTRY="${DEVOPS_DOCKER_URL}" + SRC_PATH="nginx-azure-lb/nginxaas-operator/nginxaas-operator" + SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") + SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" + + DST_REGISTRY="docker.io" + DST_PATH="nginx/nginxaas-operator" + DST_TAG="${CI_COMMIT_TAG}" + DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" + + devops.docker.login + docker pull "${SRC_IMG}" + docker tag "${SRC_IMG}" "${DST_IMG}" + + # Login to Dockerhub and push release image to it. + docker login --username "${DOCKERHUB_USERNAME}" --password "${DOCKERHUB_PASSWORD}" "${DST_REGISTRY}" + docker push "${DST_IMG}" +} + + +help_text() { + echo "Usage: $(basename $0) " +} + +parse_args() { + if [[ "$#" -ne 1 ]]; then + help_text + exit 0 + fi + + artifact="${1}" + valid_artifact="(docker-image|helm-chart)" + valid_artifact_pttn="^${valid_artifact}$" + if ! [[ "${artifact}" =~ $valid_artifact_pttn ]]; then + echo "Invalid artifact type. Valid artifact types: $valid_artifact" + help_text + exit 1 + fi +} + +main() { + if [[ "${CI}" != "true" ]]; then + echo "This script is meant to be run in the CI." + exit 1 + fi + pttn="^release-[0-9]+\.[0-9]+\.[0-9]+" + if ! [[ "${CI_COMMIT_TAG}" =~ $pttn ]]; then + echo "CI_COMMIT_TAG needs to be set to valid semver format." + exit 1 + fi + parse_args "$@" + ROOT_DIR=$(git rev-parse --show-toplevel) + source ${ROOT_DIR}/.devops.sh + "$artifact" +} + +main "$@" From 71b9abeb6f68ffcccff785bc6c2c2797e34de252 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 26 Aug 2024 16:28:36 -0700 Subject: [PATCH 022/136] Add helm chart publishing for a release This commit adds logic to publish helm charts to dockerhub using it as an OCI helm registry. Moved around common docker code into its own function to reuse logic across helm and docker publishing. Not worrying about chart versioning right now and just setting up the CI for publishing. --- scripts/release.sh | 51 ++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 48b18a8..3068b79 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,6 +3,36 @@ set -eo pipefail docker-image() { + SRC_PATH="nginx-azure-lb/nginxaas-operator/nginxaas-operator" + SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") + SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" + DST_PATH="nginx/nginxaas-operator" + DST_TAG="${CI_COMMIT_TAG}" + DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" + + docker pull "${SRC_IMG}" + docker tag "${SRC_IMG}" "${DST_IMG}" + docker push "${DST_IMG}" +} + +helm-chart() { + SRC_PATH="nginxazurelb/charts/nginx-loadbalancer-kubernetes" + SRC_TAG="0.1.0" + SRC_CHART="oci://${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" + DST_PATH="nginxinc/charts" + DST_TAG="0.1.0" + DST_CHART="oci://${DST_REGISTRY}/${DST_PATH}" + + helm pull "${SRC_CHART}" --version "${SRC_TAG}" + helm push nginx-loadbalancer-kubernetes-${DST_TAG}.tgz "${DST_CHART}" +} + + +help_text() { + echo "Usage: $(basename $0) " +} + +set_docker_common() { DOCKERHUB_USERNAME=$(devops.secret.get "kic-dockerhub-creds" | jq -r ".username") if [[ -z "${DOCKERHUB_USERNAME}" ]]; then echo "DOCKERHUB_USERNAME needs to be set." @@ -14,29 +44,13 @@ docker-image() { echo "DOCKERHUB_PASSWORD needs to be set." exit 1 fi - SRC_REGISTRY="${DEVOPS_DOCKER_URL}" - SRC_PATH="nginx-azure-lb/nginxaas-operator/nginxaas-operator" - SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") - SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" - DST_REGISTRY="docker.io" - DST_PATH="nginx/nginxaas-operator" - DST_TAG="${CI_COMMIT_TAG}" - DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" + # Login to NGINX DevOps Registry. devops.docker.login - docker pull "${SRC_IMG}" - docker tag "${SRC_IMG}" "${DST_IMG}" - - # Login to Dockerhub and push release image to it. + # Login to Dockerhub. docker login --username "${DOCKERHUB_USERNAME}" --password "${DOCKERHUB_PASSWORD}" "${DST_REGISTRY}" - docker push "${DST_IMG}" -} - - -help_text() { - echo "Usage: $(basename $0) " } parse_args() { @@ -68,6 +82,7 @@ main() { parse_args "$@" ROOT_DIR=$(git rev-parse --show-toplevel) source ${ROOT_DIR}/.devops.sh + set_docker_common "$artifact" } From 86f8dda2d330382e823c7511a40408047c24bce1 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 26 Aug 2024 16:33:02 -0700 Subject: [PATCH 023/136] Set up CI to do helm+docker releases --- .gitlab-ci.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d4c593b..fbc5fcb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -189,12 +189,24 @@ tag: when: never - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' -dockerhub-push: +.release-common: stage: release image: $DEVTOOLS_IMG extends: - .devops-docker-cicd - script: - - ./scripts/release.sh docker-image rules: - if: '$CI_COMMIT_TAG =~ /^release-[\d]+\.[\d]+\.[\d]+/' + +dockerhub-image-release: + extends: + - .devops-docker-cicd + - .release-common + script: + - ./scripts/release.sh docker-image + +dockerhub-helm-release: + extends: + - .devops-docker-cicd + - .release-common + script: + - ./scripts/release.sh helm-chart From 1f7ff6a0506a308bf3070a7a323d55407817152c Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 16 Sep 2024 09:12:00 -0600 Subject: [PATCH 024/136] NLB-5549 Translator allows hyphens in upstream name Previously, owing to a bug, if the name of the upstream included hyphens it would be rejected by the operator. --- internal/translation/translator.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 9dad7f8..fe8532c 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -79,15 +79,14 @@ func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamSe // getContextAndUpstreamName returns the nginx context being supplied by the port (either "http" or "stream") // and the upstream name. func getContextAndUpstreamName(port v1.ServicePort) (clientType string, appName string, err error) { - parts := strings.Split(port.Name, "-") - if len(parts) != 2 { + context, upstreamName, found := strings.Cut(port.Name, "-") + switch { + case !found: return clientType, appName, fmt.Errorf("ignoring port %s because it is not in the format [http|stream]-{upstreamName}", port.Name) - } - - if parts[0] != "http" && parts[0] != "stream" { + case context != "http" && context != "stream": return clientType, appName, fmt.Errorf("port name %s does not include \"http\" or \"stream\" context", port.Name) + default: + return context, upstreamName, nil } - - return parts[0], parts[1], nil } From 26c861b87491cf9193a71d8f4bd0fce1130e249e Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 12 Sep 2024 09:24:16 -0700 Subject: [PATCH 025/136] Fix path to pull helm charts Helm charts are published under `nginx-azure-lb` alongside docker images for all of our services. In addition to the above, charts are published keeping in mind that each feature branch iteration can produce a helm chart. While doing a release, we need to pull in the main chart and then re-tag it to push to docker hub. --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index 3068b79..089bb4b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -16,7 +16,7 @@ docker-image() { } helm-chart() { - SRC_PATH="nginxazurelb/charts/nginx-loadbalancer-kubernetes" + SRC_PATH="nginx-azure-lb/nginxaas-operator/charts/main/nginx-loadbalancer-kubernetes" SRC_TAG="0.1.0" SRC_CHART="oci://${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" DST_PATH="nginxinc/charts" From 1f669dd2c45fca8807e5115439140ec153873a8b Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 12 Sep 2024 09:36:55 -0700 Subject: [PATCH 026/136] Keep major version to 0 We are going to release with `0.x.y` and keeping the internal version inline with what will get published out will reduce confusion. --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index e0212c0..a8eecae 100644 --- a/version +++ b/version @@ -1 +1 @@ -export VERSION="1.$(date +"%Y%m%d").${CI_PIPELINE_ID:-0}" +export VERSION="0.$(date +"%Y%m%d").${CI_PIPELINE_ID:-0}" From 8432c0abe987bba665b82be7695c3481d6fe718d Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 19 Sep 2024 04:18:13 -0700 Subject: [PATCH 027/136] Fix how chart gets pushed to remote Version is not specified like we usually specify docker tags. Instead, the version is checked with what's specified inside the chart. --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index 089bb4b..9d4bd40 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -18,7 +18,7 @@ docker-image() { helm-chart() { SRC_PATH="nginx-azure-lb/nginxaas-operator/charts/main/nginx-loadbalancer-kubernetes" SRC_TAG="0.1.0" - SRC_CHART="oci://${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" + SRC_CHART="oci://${SRC_REGISTRY}/${SRC_PATH}" DST_PATH="nginxinc/charts" DST_TAG="0.1.0" DST_CHART="oci://${DST_REGISTRY}/${DST_PATH}" From d70bfb64f5e7ec3427c0a49f43f2461050e158e5 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 19 Sep 2024 04:25:13 -0700 Subject: [PATCH 028/136] Fix src image tag Images are tagged as "main-". --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index 9d4bd40..42f5073 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -5,7 +5,7 @@ set -eo pipefail docker-image() { SRC_PATH="nginx-azure-lb/nginxaas-operator/nginxaas-operator" SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") - SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:${SRC_TAG}" + SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:main-${SRC_TAG}" DST_PATH="nginx/nginxaas-operator" DST_TAG="${CI_COMMIT_TAG}" DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" From 3a134e8edd0d9825d2d18566e10800d6cb90a8a8 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 20 Sep 2024 16:26:55 -0600 Subject: [PATCH 029/136] Use semantic versioning This repository is going to produce artifacts that will be available publicly and the end users will care about semantic versioning. We need to be able to map a public facing version to internally produced artifacts easily and having semver internally eases that work. This commit does not enforce the versioning but adds a version file that has the semver, which will be used to version the product. We can follow a workflow where during release time, we cut a release, which creates a tag and we retag existing dev artifacts to be shipped as an official artifact. --- Makefile | 2 +- scripts/build.sh | 2 +- scripts/docker.sh | 2 +- scripts/publish-helm.sh | 6 +++--- version | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 2795d83..f0c88a8 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ BUILD_DIR = build export BUILD_DIR RESULTS_DIR = results export RESULTS_DIR -VERSION = $(shell bash -c 'source version; echo $$VERSION') +VERSION = $(shell cat version) export VERSION DOCKER_REGISTRY ?= local diff --git a/scripts/build.sh b/scripts/build.sh index b5be0ed..aafb418 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -9,7 +9,7 @@ if [[ -z $CI_COMMIT_SHORT_SHA ]]; then CI_COMMIT_SHORT_SHA=$(git rev-parse --short=8 HEAD) fi if [[ -z $VERSION ]]; then - VERSION=$(source version;echo "$VERSION") + VERSION=$(cat version) fi if [ "$os" == "linux" ]; then diff --git a/scripts/docker.sh b/scripts/docker.sh index 4193f55..af473f1 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -81,6 +81,6 @@ fi repo="${DEVOPS_DOCKER_URL}/nginx-azure-lb/${CI_PROJECT_NAME}/$image" # shellcheck source=/dev/null # shellcheck disable=SC2153 -version=$(source "${ROOT_DIR}/version";echo "$VERSION") +version=$(cat version) "$action" diff --git a/scripts/publish-helm.sh b/scripts/publish-helm.sh index 45e94aa..7d3de56 100755 --- a/scripts/publish-helm.sh +++ b/scripts/publish-helm.sh @@ -5,8 +5,8 @@ set -eo pipefail ROOT_DIR=$(git rev-parse --show-toplevel) publish_helm() { - pkg="nginx-loadbalancer-kubernetes-${VERSION}.tgz" - helm package --version "${VERSION}" --app-version "${VERSION}" charts/nlk + pkg="nginx-loadbalancer-kubernetes-${version}.tgz" + helm package --version "${version}" --app-version "${version}" charts/nlk helm push "${pkg}" "${repo}" } @@ -34,6 +34,6 @@ fi repo="oci://${DEVOPS_DOCKER_URL}/nginx-azure-lb/${CI_PROJECT_NAME}/charts/${CI_COMMIT_REF_SLUG}" # shellcheck source=/dev/null # shellcheck disable=SC2153 -version=$(source "${ROOT_DIR}/version";echo "$VERSION") +version=$(cat version) publish_helm diff --git a/version b/version index a8eecae..6e8bf73 100644 --- a/version +++ b/version @@ -1 +1 @@ -export VERSION="0.$(date +"%Y%m%d").${CI_PIPELINE_ID:-0}" +0.1.0 From d972271408191bfca122335d69e752a17af9e3b2 Mon Sep 17 00:00:00 2001 From: sarna Date: Sat, 21 Sep 2024 17:56:38 -0600 Subject: [PATCH 030/136] Remove auto tagging To follow a simple process of mapping external releases to internal artifacts, a simple semver will help. This means that auto-creating tags on merges to main from the semver will lead to confusion. --- .gitlab-ci.yml | 12 ------------ scripts/release-tag.sh | 18 ------------------ 2 files changed, 30 deletions(-) delete mode 100755 scripts/release-tag.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fbc5fcb..fe36d93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -177,18 +177,6 @@ checkmarx-scan: needs: [] allow_failure: true -tag: - stage: release - image: $DEVTOOLS_IMG - extends: - - .default-runner - script: - - ./scripts/release-tag.sh - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: never - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - .release-common: stage: release image: $DEVTOOLS_IMG diff --git a/scripts/release-tag.sh b/scripts/release-tag.sh deleted file mode 100755 index 8c6d639..0000000 --- a/scripts/release-tag.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -### This script should only run during master pipelines ### -### This script will create tags for the master commit using the current version ### - -set -eo pipefail - -if [ "$CI_COMMIT_REF_NAME" = "main" ]; then - # shellcheck source=/dev/null - version_tag=v$(source version;echo "$VERSION") - - curl -s \ - --request POST \ - --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN_RW}" \ - "https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/repository/tags" \ - --form "tag_name=$version_tag" \ - --form "ref=$CI_COMMIT_SHA" -fi From f1d7238bf1b8bd9fc77fb06bda5478c9d30b7e90 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 23 Sep 2024 11:52:41 -0700 Subject: [PATCH 031/136] Allow CI to run on tags --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fe36d93..d3d94f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,7 @@ workflow: - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS when: never - - if: $CI_COMMIT_BRANCH + - if: '$CI_COMMIT_BRANCH || $CI_COMMIT_TAG' .import-devops-core-services: &import-devops-core-services | source ${CI_PROJECT_DIR}/.devops.sh From b49b4adc0ff76a5d47b8a7905917cea84060caa2 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 23 Sep 2024 14:20:53 -0700 Subject: [PATCH 032/136] Fix path to push chart Charts go under https://hub.docker.com/orgs/nginxcharts --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index 42f5073..9f79999 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -19,7 +19,7 @@ helm-chart() { SRC_PATH="nginx-azure-lb/nginxaas-operator/charts/main/nginx-loadbalancer-kubernetes" SRC_TAG="0.1.0" SRC_CHART="oci://${SRC_REGISTRY}/${SRC_PATH}" - DST_PATH="nginxinc/charts" + DST_PATH="nginxcharts" DST_TAG="0.1.0" DST_CHART="oci://${DST_REGISTRY}/${DST_PATH}" From f3e233e4b958bdc935742d254f42d1bdeae46f0b Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 11:05:05 -0700 Subject: [PATCH 033/136] Rename the chart to nginxaas-operator While a cosmetic change, it does impact how dokerhub repo needs to be setup to publish the helm chart. In addition to that, it impacts what the user see on k8s itself and it should not be nginx-loadbalancer-kubernetes as that is confusing. --- charts/nlk/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index e82eaab..1f986a3 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 0.1.0 description: NGINX LoadBalancer for Kubernetes -name: nginx-loadbalancer-kubernetes +name: nginxaas-operator home: https://github.com/nginxinc/nginx-loadbalancer-kubernetes icon: https://raw.githubusercontent.com/nginxinc/nginx-loadbalancer-kubernetes/main/nlk-logo.svg keywords: From 2cde46d885f1e279c51a5409ac11d5f16fadfcf4 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 13:04:36 -0700 Subject: [PATCH 034/136] Update name to be operator While the chartname (which was renamed in the prior commit) gets used for naming stuff, it makes sense to also change the name value in the chart itself to use the new name. --- charts/nlk/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 394bc1f..ab957a6 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -1,5 +1,5 @@ nlk: - name: nginx-loadbalancer-kubernetes + name: nginxaas-operator kind: deployment From e292269480736028a4267f36f4ca5747fd7ad666 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 13:06:33 -0700 Subject: [PATCH 035/136] Publish chart with new name --- scripts/publish-helm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/publish-helm.sh b/scripts/publish-helm.sh index 7d3de56..8abecc5 100755 --- a/scripts/publish-helm.sh +++ b/scripts/publish-helm.sh @@ -5,7 +5,7 @@ set -eo pipefail ROOT_DIR=$(git rev-parse --show-toplevel) publish_helm() { - pkg="nginx-loadbalancer-kubernetes-${version}.tgz" + pkg="nginxaas-operator-${version}.tgz" helm package --version "${version}" --app-version "${version}" charts/nlk helm push "${pkg}" "${repo}" } From 630179f1552568cda1be43569eedf378d638ac57 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 13:07:39 -0700 Subject: [PATCH 036/136] Use new name for publishing release artifacts Once the rename occurs, we need to update the release logic to pull the newly named chart, and push it up to dockerhub. --- scripts/release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 9f79999..aa942c8 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -16,7 +16,7 @@ docker-image() { } helm-chart() { - SRC_PATH="nginx-azure-lb/nginxaas-operator/charts/main/nginx-loadbalancer-kubernetes" + SRC_PATH="nginx-azure-lb/nginxaas-operator/charts/main/nginxaas-operator" SRC_TAG="0.1.0" SRC_CHART="oci://${SRC_REGISTRY}/${SRC_PATH}" DST_PATH="nginxcharts" @@ -24,7 +24,7 @@ helm-chart() { DST_CHART="oci://${DST_REGISTRY}/${DST_PATH}" helm pull "${SRC_CHART}" --version "${SRC_TAG}" - helm push nginx-loadbalancer-kubernetes-${DST_TAG}.tgz "${DST_CHART}" + helm push nginxaas-operator-${DST_TAG}.tgz "${DST_CHART}" } From dca1ce58d792562a21d3f9dae26dfef976679659 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 21:05:34 -0700 Subject: [PATCH 037/136] Update registry paths to dockerhub Now that official images exist on docker hub, we should use those images in our charts. --- charts/nlk/values.yaml | 36 ++++++++++++++++++------------------ version | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index ab957a6..11d082b 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -4,18 +4,18 @@ nlk: kind: deployment replicaCount: 1 - + image: - registry: ghcr.io - repository: nginxinc/nginx-loadbalancer-kubernetes + registry: registry-1.docker.io + repository: nginx/nginxaas-operator pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: latest - + tag: release-0.1.0 + imagePullSecrets: [] nameOverride: "" fullnameOverride: "" - + serviceAccount: # Specifies whether a service account should be created create: true @@ -23,13 +23,13 @@ nlk: automount: true # Annotations to add to the service account annotations: {} - + podAnnotations: {} podLabels: {} - + podSecurityContext: {} # fsGroup: 2000 - + securityContext: {} # capabilities: # drop: @@ -37,11 +37,11 @@ nlk: # readOnlyRootFilesystem: true # runAsNonRoot: true # runAsUser: 1000 - + service: type: ClusterIP port: 80 - + ingress: enabled: false className: "" @@ -57,7 +57,7 @@ nlk: # - secretName: chart-example-tls # hosts: # - chart-example.local - + resources: requests: cpu: 100m @@ -65,31 +65,31 @@ nlk: # limits: # cpu: 100m # memory: 128Mi - + autoscaling: enabled: false minReplicas: 1 maxReplicas: 3 targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 - + # Additional volumes on the output Deployment definition. volumes: [] # - name: foo # secret: # secretName: mysecret # optional: false - + # Additional volumeMounts on the output Deployment definition. volumeMounts: [] # - name: foo # mountPath: "/etc/foo" # readOnly: true - + nodeSelector: {} - + tolerations: [] - + affinity: {} config: diff --git a/version b/version index 6e8bf73..d917d3e 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.0 +0.1.2 From c25d857eead971ad11cd453e648203697500567c Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 24 Sep 2024 21:06:37 -0700 Subject: [PATCH 038/136] Update gitignore with new chart name Now that the chart has been renamed, gitignore needs to know about ignoring new charts. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 396ec9f..f434c43 100644 --- a/.gitignore +++ b/.gitignore @@ -88,4 +88,4 @@ results .go-build -nginx-loadbalancer-kubernetes-* +nginxaas-operator-* From 7dd452af0a471053d78da8104cd488d685f1bd3e Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 25 Sep 2024 00:40:39 -0700 Subject: [PATCH 039/136] Add support for image pull secrets This lets us test the operator from the private registry. Also, in case a customer does not want to pull from docker hub and instead use their own registry, they can do so and specify a pull secret for the image. --- charts/nlk/templates/nlk-deployment.yaml | 4 ++++ version | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index fb55d77..2653d0b 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -15,6 +15,10 @@ spec: labels: app: nlk spec: + {{- with .Values.nlk.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: {{ .Chart.Name }} image: {{ include "nlk.image" .}} diff --git a/version b/version index d917d3e..b1e80bb 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.2 +0.1.3 From 83ce3b6c56e53cc845632bbfc638669a8ea2b843 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 26 Sep 2024 19:13:06 -0700 Subject: [PATCH 040/136] Mount configmap as a volume With the change to make the operator read a config file (to reduce code complexity), we need to make sure that the file exists for the service to start. --- charts/nlk/templates/nlk-configmap.yaml | 12 ++++++------ charts/nlk/templates/nlk-deployment.yaml | 7 +++++++ version | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index 482b8cb..ab8977f 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -4,11 +4,11 @@ metadata: name: nlk-config namespace: nlk data: + config.yaml: | {{- if .Values.nlk.config.entries.hosts }} - nginx-hosts: "{{ .Values.nlk.config.entries.hosts }}" + nginx-hosts: "{{ .Values.nlk.config.entries.hosts }}" {{- end }} - tls-mode: "{{ index .Values.nlk.defaultTLS "tls-mode" }}" - ca-certificate: "{{ index .Values.nlk.defaultTLS "ca-certificate" }}" - client-certificate: "{{ index .Values.nlk.defaultTLS "client-certificate" }}" - log-level: "{{ .Values.nlk.logLevel }}" - + tls-mode: "{{ index .Values.nlk.defaultTLS "tls-mode" }}" + ca-certificate: "{{ index .Values.nlk.defaultTLS "ca-certificate" }}" + client-certificate: "{{ index .Values.nlk.defaultTLS "client-certificate" }}" + log-level: "{{ .Values.nlk.logLevel }}" diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 2653d0b..3b5348e 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -45,4 +45,11 @@ spec: initialDelaySeconds: {{ .Values.nlk.readyStatus.initialDelaySeconds }} periodSeconds: {{ .Values.nlk.readyStatus.periodSeconds }} {{- end }} + volumeMounts: + - name: config + mountPath: /etc/nginxaas-operator serviceAccountName: {{ include "nlk.fullname" . }} + volumes: + - name: config + configMap: + name: nlk-config diff --git a/version b/version index b1e80bb..845639e 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.3 +0.1.4 From 300ea61c2417cd804ecfae00f0372b978f0e9388 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 26 Sep 2024 19:24:21 -0700 Subject: [PATCH 041/136] Drop configmap access for operator Once the operator reads from the configmap mounted as a volume, it does not need access to the k8s API to read the config map itself. --- charts/nlk/templates/clusterrole.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/nlk/templates/clusterrole.yaml b/charts/nlk/templates/clusterrole.yaml index 4164475..0ab90bb 100644 --- a/charts/nlk/templates/clusterrole.yaml +++ b/charts/nlk/templates/clusterrole.yaml @@ -7,7 +7,6 @@ rules: - apiGroups: - "" resources: - - configmaps - nodes - secrets - services From bbf990459445145f5dccb34fcc56e796eef5651d Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 27 Sep 2024 12:40:24 -0700 Subject: [PATCH 042/136] Grant permissions to read configmaps This is needed temporarily while nlk still needs to read the configmap and the code for swapping it out with a config file does not land. The order of merges that I am thinking: - Get this MR landed. - Land testenv changes to use helm. - Land code to read settings from configfile. - Remove configmap permissions. --- charts/nlk/templates/clusterrole.yaml | 1 + version | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/nlk/templates/clusterrole.yaml b/charts/nlk/templates/clusterrole.yaml index 0ab90bb..4164475 100644 --- a/charts/nlk/templates/clusterrole.yaml +++ b/charts/nlk/templates/clusterrole.yaml @@ -7,6 +7,7 @@ rules: - apiGroups: - "" resources: + - configmaps - nodes - secrets - services diff --git a/version b/version index 845639e..9faa1b7 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.4 +0.1.5 From 1d8cf2d04fd537a8aeb8daa96d45af7ea91d5feb Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 5 Sep 2024 08:54:06 -0600 Subject: [PATCH 043/136] NLB-5342 Configuration settings now read from config file This means that we do not have to grant the operator app any permissions to access kubernetes secrets. Reading from the config file changes the application's behavior in that settings are no longer changed whenever a config map is updated at run time, as they used to be. Settings are now concurrency-safe, because they pass new values instead of a shared pointer to their consumers in separate goroutines. Settings also no longer pass the application context to consumers as this is a well-document go anti-pattern. Context should always be passed as a function parameter. Any operator modules that need access to a kubernetes client are constructed with a reference to the client, instead of gaining access to the client through settings. --- cmd/configuration-test-harness/doc.go | 1 - cmd/configuration-test-harness/main.go | 81 ---- cmd/nginx-loadbalancer-kubernetes/main.go | 46 ++- cmd/tls-config-factory-test-harness/main.go | 2 +- go.mod | 40 +- go.sum | 377 ++++-------------- internal/authentication/factory.go | 2 +- internal/authentication/factory_test.go | 21 +- internal/certification/certificates_test.go | 5 +- internal/communication/factory.go | 4 +- internal/communication/factory_test.go | 17 +- internal/communication/roundtripper_test.go | 21 +- internal/configuration/configuration_test.go | 52 +++ internal/configuration/settings.go | 277 ++----------- internal/configuration/testdata/test.yaml | 11 + internal/observation/handler.go | 6 +- internal/observation/handler_test.go | 21 +- internal/observation/watcher.go | 53 +-- internal/observation/watcher_test.go | 5 +- internal/synchronization/synchronizer.go | 4 +- internal/synchronization/synchronizer_test.go | 74 ++-- 21 files changed, 355 insertions(+), 765 deletions(-) delete mode 100644 cmd/configuration-test-harness/doc.go delete mode 100644 cmd/configuration-test-harness/main.go create mode 100644 internal/configuration/configuration_test.go create mode 100644 internal/configuration/testdata/test.yaml diff --git a/cmd/configuration-test-harness/doc.go b/cmd/configuration-test-harness/doc.go deleted file mode 100644 index 06ab7d0..0000000 --- a/cmd/configuration-test-harness/doc.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/cmd/configuration-test-harness/main.go b/cmd/configuration-test-harness/main.go deleted file mode 100644 index 5079a9d..0000000 --- a/cmd/configuration-test-harness/main.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "path/filepath" - - configuration2 "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/sirupsen/logrus" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" -) - -func main() { - logrus.SetLevel(logrus.DebugLevel) - err := run() - if err != nil { - logrus.Fatal(err) - } -} - -func run() error { - logrus.Info("configuration-test-harness::run") - - ctx := context.Background() - var err error - - k8sClient, err := buildKubernetesClient() - if err != nil { - return fmt.Errorf(`error building a Kubernetes client: %w`, err) - } - - configuration, err := configuration2.NewSettings(ctx, k8sClient) - if err != nil { - return fmt.Errorf(`error occurred creating configuration: %w`, err) - } - - err = configuration.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing configuration: %w`, err) - } - - go configuration.Run() - - <-ctx.Done() - - return err -} - -func buildKubernetesClient() (*kubernetes.Clientset, error) { - logrus.Debug("Watcher::buildKubernetesClient") - - var kubeconfig *string - var k8sConfig *rest.Config - - k8sConfig, err := rest.InClusterConfig() - if errors.Is(err, rest.ErrNotInCluster) { - if home := homedir.HomeDir(); home != "" { - path := filepath.Join(home, ".kube", "config") - kubeconfig = &path - - k8sConfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) - if err != nil { - return nil, fmt.Errorf(`error occurred building the kubeconfig: %w`, err) - } - } else { - return nil, fmt.Errorf(`not running in a Cluster: %w`, err) - } - } else if err != nil { - return nil, fmt.Errorf(`error occurred getting the Cluster config: %w`, err) - } - - client, err := kubernetes.NewForConfig(k8sConfig) - if err != nil { - return nil, fmt.Errorf(`error occurred creating a client: %w`, err) - } - return client, nil -} diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 89f764f..f8473c3 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -35,17 +35,12 @@ func run() error { return fmt.Errorf(`error building a Kubernetes client: %w`, err) } - settings, err := configuration.NewSettings(ctx, k8sClient) + settings, err := configuration.Read("config.yaml", "/etc/nginxaas-operator") if err != nil { - return fmt.Errorf(`error occurred creating settings: %w`, err) + return fmt.Errorf(`error occurred accessing configuration: %w`, err) } - err = settings.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing settings: %w`, err) - } - - go settings.Run() + setLogLevel(settings.LogLevel) synchronizerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) @@ -58,12 +53,12 @@ func run() error { handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue) - watcher, err := observation.NewWatcher(settings, handler) + watcher, err := observation.NewWatcher(settings, handler, k8sClient) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } - err = watcher.Initialize() + err = watcher.Initialize(ctx) if err != nil { return fmt.Errorf(`error occurred initializing the watcher: %w`, err) } @@ -74,7 +69,7 @@ func run() error { probeServer := probation.NewHealthServer() probeServer.Start() - err = watcher.Watch() + err = watcher.Watch(ctx) if err != nil { return fmt.Errorf(`error occurred watching for events: %w`, err) } @@ -83,6 +78,35 @@ func run() error { return nil } +func setLogLevel(logLevel string) { + logrus.Debugf("Settings::setLogLevel: %s", logLevel) + switch logLevel { + case "panic": + logrus.SetLevel(logrus.PanicLevel) + + case "fatal": + logrus.SetLevel(logrus.FatalLevel) + + case "error": + logrus.SetLevel(logrus.ErrorLevel) + + case "warn": + logrus.SetLevel(logrus.WarnLevel) + + case "info": + logrus.SetLevel(logrus.InfoLevel) + + case "debug": + logrus.SetLevel(logrus.DebugLevel) + + case "trace": + logrus.SetLevel(logrus.TraceLevel) + + default: + logrus.SetLevel(logrus.WarnLevel) + } +} + func buildKubernetesClient() (*kubernetes.Clientset, error) { logrus.Debug("Watcher::buildKubernetesClient") k8sConfig, err := rest.InClusterConfig() diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 51f1d1d..7b853e0 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -31,7 +31,7 @@ func main() { logrus.Infof("\n\n\t*** Building TLS config for <<< %s >>>\n\n", name) - tlsConfig, err := authentication.NewTLSConfig(&settings.Settings) + tlsConfig, err := authentication.NewTLSConfig(settings.Settings) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index 118de5f..056c7e2 100644 --- a/go.mod +++ b/go.mod @@ -11,42 +11,60 @@ toolchain go1.21.4 require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/sirupsen/logrus v1.9.0 + github.com/spf13/viper v1.19.0 + github.com/stretchr/testify v1.9.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.19.14 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.6 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.80.1 // indirect diff --git a/go.sum b/go.sum index 46a175a..098adea 100644 --- a/go.sum +++ b/go.sum @@ -1,63 +1,26 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -69,86 +32,59 @@ github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -164,296 +100,157 @@ github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -463,12 +260,7 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= @@ -481,9 +273,6 @@ k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+O k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go index 32add62..a51d7ab 100644 --- a/internal/authentication/factory.go +++ b/internal/authentication/factory.go @@ -18,7 +18,7 @@ import ( "github.com/sirupsen/logrus" ) -func NewTLSConfig(settings *configuration.Settings) (*tls.Config, error) { +func NewTLSConfig(settings configuration.Settings) (*tls.Config, error) { logrus.Debugf("authentication::NewTLSConfig Creating TLS config for mode: '%s'", settings.TLSMode) switch settings.TLSMode { diff --git a/internal/authentication/factory_test.go b/internal/authentication/factory_test.go index e9015c0..6b5fcaf 100644 --- a/internal/authentication/factory_test.go +++ b/internal/authentication/factory_test.go @@ -20,9 +20,8 @@ const ( func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { t.Parallel() - settings := configuration.Settings{} - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(configuration.Settings{}) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -50,7 +49,7 @@ func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { }, } - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -84,7 +83,7 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -108,7 +107,7 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -133,7 +132,7 @@ func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { }, } - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -168,7 +167,7 @@ func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -193,7 +192,7 @@ func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } @@ -209,7 +208,7 @@ func TestTlsFactory_CaTlsMode(t *testing.T) { TLSMode: configuration.CertificateAuthorityTLS, } - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -245,7 +244,7 @@ func TestTlsFactory_CaMtlsMode(t *testing.T) { }, } - tlsConfig, err := NewTLSConfig(&settings) + tlsConfig, err := NewTLSConfig(settings) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -280,7 +279,7 @@ func TestTlsFactory_CaMtlsModeClientCertificateError(t *testing.T) { }, } - _, err := NewTLSConfig(&settings) + _, err := NewTLSConfig(settings) if err == nil { t.Fatalf(`Expected an error`) } diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go index 89b21bf..901964a 100644 --- a/internal/certification/certificates_test.go +++ b/internal/certification/certificates_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -94,9 +95,7 @@ func TestCertificates_ExerciseHandlers(t *testing.T) { //nolint:govet,staticcheck go func() { err := certificates.Run() - if err != nil { - t.Fatalf("error running Certificates: %v", err) - } + assert.NoError(t, err, "expected no error running certificates") }() cache.WaitForCacheSync(ctx.Done(), certificates.informer.HasSynced) diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 64ffe68..2a3c09a 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -19,7 +19,7 @@ import ( // NewHTTPClient is a factory method to create a new Http Client with a default configuration. // RoundTripper is a wrapper around the default net/communication Transport to add additional headers, in this case, // the Headers are configured for JSON. -func NewHTTPClient(settings *configuration.Settings) (*netHttp.Client, error) { +func NewHTTPClient(settings configuration.Settings) (*netHttp.Client, error) { headers := NewHeaders(settings.APIKey) tlsConfig := NewTLSConfig(settings) transport := NewTransport(tlsConfig) @@ -49,7 +49,7 @@ func NewHeaders(apiKey string) []string { // NewTLSConfig is a factory method to create a new basic Tls Config. // More attention should be given to the use of `InsecureSkipVerify: true`, as it is not recommended for production use. -func NewTLSConfig(settings *configuration.Settings) *tls.Config { +func NewTLSConfig(settings configuration.Settings) *tls.Config { tlsConfig, err := authentication.NewTLSConfig(settings) if err != nil { logrus.Warnf("Failed to create TLS config: %v", err) diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index 398bb6c..65f5e5b 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -6,21 +6,13 @@ package communication import ( - "context" "testing" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "k8s.io/client-go/kubernetes/fake" ) func TestNewHTTPClient(t *testing.T) { t.Parallel() - k8sClient := fake.NewSimpleClientset() - settings, err := configuration.NewSettings(context.Background(), k8sClient) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - client, err := NewHTTPClient(settings) + + client, err := NewHTTPClient(defaultSettings()) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -79,9 +71,8 @@ func TestNewHeadersWithNoAPIKey(t *testing.T) { func TestNewTransport(t *testing.T) { t.Parallel() - k8sClient := fake.NewSimpleClientset() - settings, _ := configuration.NewSettings(context.Background(), k8sClient) - config := NewTLSConfig(settings) + + config := NewTLSConfig(defaultSettings()) transport := NewTransport(config) if transport == nil { diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index abee455..9913600 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -7,20 +7,17 @@ package communication import ( "bytes" - "context" netHttp "net/http" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "k8s.io/client-go/kubernetes/fake" ) func TestNewRoundTripper(t *testing.T) { t.Parallel() - k8sClient := fake.NewSimpleClientset() - settings, _ := configuration.NewSettings(context.Background(), k8sClient) + headers := NewHeaders("fakeKey") - transport := NewTransport(NewTLSConfig(settings)) + transport := NewTransport(NewTLSConfig(defaultSettings())) roundTripper := NewRoundTripper(headers, transport) if roundTripper == nil { @@ -54,13 +51,9 @@ func TestNewRoundTripper(t *testing.T) { func TestRoundTripperRoundTrip(t *testing.T) { t.Parallel() - k8sClient := fake.NewSimpleClientset() - settings, err := configuration.NewSettings(context.Background(), k8sClient) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } + headers := NewHeaders("fakeKey") - transport := NewTransport(NewTLSConfig(settings)) + transport := NewTransport(NewTLSConfig(defaultSettings())) roundTripper := NewRoundTripper(headers, transport) request, err := NewRequest("GET", "http://example.com", nil) @@ -95,3 +88,9 @@ func NewRequest(method string, url string, body []byte) (*netHttp.Request, error return request, nil } + +func defaultSettings() configuration.Settings { + return configuration.Settings{ + TLSMode: configuration.NoTLS, + } +} diff --git a/internal/configuration/configuration_test.go b/internal/configuration/configuration_test.go new file mode 100644 index 0000000..9694989 --- /dev/null +++ b/internal/configuration/configuration_test.go @@ -0,0 +1,52 @@ +package configuration_test + +import ( + "testing" + "time" + + "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + + "github.com/stretchr/testify/require" +) + +func TestConfiguration(t *testing.T) { + t.Parallel() + expectedSettings := configuration.Settings{ + LogLevel: "warn", + NginxPlusHosts: []string{"https://10.0.0.1:9000/api"}, + TLSMode: configuration.NoTLS, + Certificates: &certification.Certificates{ + CaCertificateSecretKey: "fakeCAKey", + ClientCertificateSecretKey: "fakeCertKey", + }, + Handler: configuration.HandlerSettings{ + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-handler", + }, + }, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + Watcher: configuration.WatcherSettings{ + ResyncPeriod: 0, + ServiceAnnotation: "fakeServiceMatch", + }, + } + + settings, err := configuration.Read("test", "./testdata") + require.NoError(t, err) + require.Equal(t, expectedSettings, settings) +} diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index 400b311..91ff27c 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -6,21 +6,14 @@ package configuration import ( - "context" "encoding/base64" "fmt" - "os" - "strings" "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" + + "github.com/spf13/viper" ) const ( @@ -111,8 +104,8 @@ type SynchronizerSettings struct { // Settings contains the configuration values needed by the application. type Settings struct { - // Context is the context used to control the application. - Context context.Context + // LogLevel is the user-specified log level. Defaults to warn. + LogLevel string // NginxPlusHosts is a list of Nginx Plus hosts that will be used to update the Border Servers. NginxPlusHosts []string @@ -127,15 +120,6 @@ type Settings struct { // Certificates is the object used to retrieve the certificates and keys used to communicate with the Border Servers. Certificates *certification.Certificates - // K8sClient is the Kubernetes client used to communicate with the Kubernetes API. - K8sClient kubernetes.Interface - - // informer is the SharedInformer used to watch for changes to the ConfigMap . - informer cache.SharedInformer - - // eventHandlerRegistration is the object used to track the event handlers with the SharedInformer. - eventHandlerRegistration cache.ResourceEventHandlerRegistration - // Handler contains the configuration values needed by the Handler. Handler HandlerSettings @@ -146,17 +130,41 @@ type Settings struct { Watcher WatcherSettings } -// NewSettings creates a new Settings object with default values. -func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings, error) { - // get base64 encoded version of raw api key set by user - apiKey := base64.StdEncoding.EncodeToString([]byte(os.Getenv("NGINXAAS_DATAPLANE_API_KEY"))) - - settings := &Settings{ - Context: ctx, - K8sClient: k8sClient, - TLSMode: NoTLS, - APIKey: apiKey, - Certificates: nil, +// Read parses all the config and returns the values +func Read(configName, configPath string) (s Settings, err error) { + v := viper.New() + v.SetConfigName(configName) + v.SetConfigType("yaml") + v.AddConfigPath(configPath) + if err = v.ReadInConfig(); err != nil { + return s, err + } + + if err = v.BindEnv("NGINXAAS_DATAPLANE_API_KEY"); err != nil { + return s, err + } + + tlsMode := NoTLS + if t, err := validateTLSMode(v.GetString("tls-mode")); err != nil { + logrus.Errorf("could not validate tls mode: %v", err) + } else { + tlsMode = t + } + + serviceAnnotation := DefaultServiceAnnotation + if sa := v.GetString(ServiceAnnotationMatchKey); sa != "" { + serviceAnnotation = sa + } + + return Settings{ + LogLevel: v.GetString("log-level"), + NginxPlusHosts: v.GetStringSlice("nginx-hosts"), + TLSMode: tlsMode, + APIKey: base64.StdEncoding.EncodeToString([]byte(v.GetString("NGINXAAS_DATAPLANE_API_KEY"))), + Certificates: &certification.Certificates{ + CaCertificateSecretKey: v.GetString("ca-certificate"), + ClientCertificateSecretKey: v.GetString("client-certificate"), + }, Handler: HandlerSettings{ RetryCount: 5, Threads: 1, @@ -179,216 +187,15 @@ func NewSettings(ctx context.Context, k8sClient kubernetes.Interface) (*Settings }, Watcher: WatcherSettings{ ResyncPeriod: 0, - ServiceAnnotation: DefaultServiceAnnotation, + ServiceAnnotation: serviceAnnotation, }, - } - - return settings, nil -} - -// Initialize initializes the Settings object. Sets up a SharedInformer to watch for changes to the ConfigMap. -// This method must be called before the Run method. -func (s *Settings) Initialize() error { - logrus.Info("Settings::Initialize") - - var err error - - certificates := certification.NewCertificates(s.Context, s.K8sClient) - - err = certificates.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing certificates: %w`, err) - } - - s.Certificates = certificates - - go certificates.Run() //nolint:errcheck - - logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieving nlk-config ConfigMap") - configMap, err := s.K8sClient.CoreV1().ConfigMaps(ConfigMapsNamespace).Get( - s.Context, "nlk-config", metav1.GetOptions{}, - ) - if err != nil { - return err - } - - s.handleUpdateEvent(nil, configMap) - logrus.Debug(">>>>>>>>>> Settings::Initialize: retrieved nlk-config ConfigMap") - - informer := s.buildInformer() - - s.informer = informer - - err = s.initializeEventListeners() - if err != nil { - return fmt.Errorf(`error occurred initializing event listeners: %w`, err) - } - - return nil -} - -// Run starts the SharedInformer and waits for the Context to be canceled. -func (s *Settings) Run() { - logrus.Debug("Settings::Run") - - defer utilruntime.HandleCrash() - - go s.informer.Run(s.Context.Done()) - - <-s.Context.Done() -} - -func (s *Settings) buildInformer() cache.SharedInformer { - options := informers.WithNamespace(ConfigMapsNamespace) - factory := informers.NewSharedInformerFactoryWithOptions(s.K8sClient, ResyncPeriod, options) - informer := factory.Core().V1().ConfigMaps().Informer() - - return informer -} - -func (s *Settings) initializeEventListeners() error { - logrus.Debug("Settings::initializeEventListeners") - - var err error - - handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: s.handleAddEvent, - UpdateFunc: s.handleUpdateEvent, - DeleteFunc: s.handleDeleteEvent, - } - - s.eventHandlerRegistration, err = s.informer.AddEventHandler(handlers) - if err != nil { - return fmt.Errorf(`error occurred registering event handlers: %w`, err) - } - - return nil + }, nil } -func (s *Settings) handleAddEvent(obj interface{}) { - logrus.Debug("Settings::handleAddEvent") - - if _, yes := isOurConfig(obj); yes { - s.handleUpdateEvent(nil, obj) - } -} - -func (s *Settings) handleDeleteEvent(obj interface{}) { - logrus.Debug("Settings::handleDeleteEvent") - - if _, yes := isOurConfig(obj); yes { - s.updateHosts([]string{}) - } -} - -func (s *Settings) handleUpdateEvent(_ interface{}, newValue interface{}) { - logrus.Debug("Settings::handleUpdateEvent") - - configMap, yes := isOurConfig(newValue) - if !yes { - return - } - - hosts, found := configMap.Data["nginx-hosts"] - if found { - newHosts := s.parseHosts(hosts) - s.updateHosts(newHosts) - } else { - logrus.Warnf("Settings::handleUpdateEvent: nginx-hosts key not found in ConfigMap") - } - - tlsMode, err := validateTLSMode(configMap) - if err != nil { - // NOTE: the TLSMode defaults to NoTLS on startup, or the last known good value if previously set. - logrus.Errorf( - "Error with configured TLS Mode. TLS Mode has NOT been changed. The current mode is: '%v'. Error: %v. ", - s.TLSMode, err, - ) - } else { - s.TLSMode = tlsMode - } - - caCertificateSecretKey, found := configMap.Data["ca-certificate"] - if found { - s.Certificates.CaCertificateSecretKey = caCertificateSecretKey - logrus.Debugf("Settings::handleUpdateEvent: ca-certificate: %s", s.Certificates.CaCertificateSecretKey) - } else { - s.Certificates.CaCertificateSecretKey = "" - logrus.Warnf("Settings::handleUpdateEvent: ca-certificate key not found in ConfigMap") - } - - clientCertificateSecretKey, found := configMap.Data["client-certificate"] - if found { - s.Certificates.ClientCertificateSecretKey = clientCertificateSecretKey - logrus.Debugf("Settings::handleUpdateEvent: client-certificate: %s", s.Certificates.ClientCertificateSecretKey) - } else { - s.Certificates.ClientCertificateSecretKey = "" - logrus.Warnf("Settings::handleUpdateEvent: client-certificate key not found in ConfigMap") - } - - if serviceAnnotation, found := configMap.Data[ServiceAnnotationMatchKey]; found { - s.Watcher.ServiceAnnotation = serviceAnnotation - } else { - s.Watcher.ServiceAnnotation = DefaultServiceAnnotation - } - logrus.Debugf("Settings::handleUpdateEvent: %s: %s", ServiceAnnotationMatchKey, s.Watcher.ServiceAnnotation) - - setLogLevel(configMap.Data["log-level"]) - - logrus.Debugf("Settings::handleUpdateEvent: \n\tHosts: %v,\n\tSettings: %v ", s.NginxPlusHosts, configMap) -} - -func validateTLSMode(configMap *corev1.ConfigMap) (TLSMode, error) { - tlsConfigMode, tlsConfigModeFound := configMap.Data["tls-mode"] - if !tlsConfigModeFound { - return NoTLS, fmt.Errorf(`tls-mode key not found in ConfigMap`) - } - +func validateTLSMode(tlsConfigMode string) (TLSMode, error) { if tlsMode, tlsModeFound := TLSModeMap[tlsConfigMode]; tlsModeFound { return tlsMode, nil } return NoTLS, fmt.Errorf(`invalid tls-mode value: %s`, tlsConfigMode) } - -func (s *Settings) parseHosts(hosts string) []string { - return strings.Split(hosts, ",") -} - -func (s *Settings) updateHosts(hosts []string) { - s.NginxPlusHosts = hosts -} - -func isOurConfig(obj interface{}) (*corev1.ConfigMap, bool) { - configMap, ok := obj.(*corev1.ConfigMap) - return configMap, ok && configMap.Name == ConfigMapName && configMap.Namespace == ConfigMapsNamespace -} - -func setLogLevel(logLevel string) { - logrus.Debugf("Settings::setLogLevel: %s", logLevel) - switch logLevel { - case "panic": - logrus.SetLevel(logrus.PanicLevel) - - case "fatal": - logrus.SetLevel(logrus.FatalLevel) - - case "error": - logrus.SetLevel(logrus.ErrorLevel) - - case "warn": - logrus.SetLevel(logrus.WarnLevel) - - case "info": - logrus.SetLevel(logrus.InfoLevel) - - case "debug": - logrus.SetLevel(logrus.DebugLevel) - - case "trace": - logrus.SetLevel(logrus.TraceLevel) - - default: - logrus.SetLevel(logrus.WarnLevel) - } -} diff --git a/internal/configuration/testdata/test.yaml b/internal/configuration/testdata/test.yaml new file mode 100644 index 0000000..717dcdb --- /dev/null +++ b/internal/configuration/testdata/test.yaml @@ -0,0 +1,11 @@ +ca-certificate: fakeCAKey +client-certificate: fakeCertKey +log-level: warn +nginx-hosts: https://10.0.0.1:9000/api +tls-mode: no-tls +service-annotation-match: fakeServiceMatch +creationTimestamp: "2024-09-04T17:59:20Z" +name: nlk-config +namespace: nlk +resourceVersion: "5909" +uid: 66d49974-49d6-4ad8-8135-dcebda7b5c9e diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 5584939..bd823f6 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -19,7 +19,6 @@ import ( // HandlerInterface is the interface for the event handler type HandlerInterface interface { - // AddRateLimitedEvent defines the interface for adding an event to the event queue AddRateLimitedEvent(event *core.Event) @@ -35,12 +34,11 @@ type HandlerInterface interface { // The translation process may result in multiple events being generated. This fan-out mainly supports the differences // in NGINX Plus API calls for creating/updating Upstreams and deleting Upstreams. type Handler struct { - // eventQueue is the queue used to store events eventQueue workqueue.RateLimitingInterface // settings is the configuration settings - settings *configuration.Settings + settings configuration.Settings // synchronizer is the synchronizer used to synchronize the internal representation with a Border Server synchronizer synchronization.Interface @@ -48,7 +46,7 @@ type Handler struct { // NewHandler creates a new event handler func NewHandler( - settings *configuration.Settings, + settings configuration.Settings, synchronizer synchronization.Interface, eventQueue workqueue.RateLimitingInterface, ) *Handler { diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index 72b6d8f..ba4add1 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -6,23 +6,17 @@ package observation import ( - "context" - "fmt" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" v1 "k8s.io/api/core/v1" - "k8s.io/client-go/util/workqueue" ) func TestHandler_AddsEventToSynchronizer(t *testing.T) { t.Parallel() - _, _, synchronizer, handler, err := buildHandler() - if err != nil { - t.Errorf(`should have been no error, %v`, err) - } + synchronizer, handler := buildHandler() event := &core.Event{ Type: core.Created, @@ -47,19 +41,12 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { } func buildHandler() ( - *configuration.Settings, - workqueue.RateLimitingInterface, - *mocks.MockSynchronizer, *Handler, error, + *mocks.MockSynchronizer, *Handler, ) { - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - return nil, nil, nil, nil, fmt.Errorf(`should have been no error, %v`, err) - } - eventQueue := &mocks.MockRateLimiter{} synchronizer := &mocks.MockSynchronizer{} - handler := NewHandler(settings, synchronizer, eventQueue) + handler := NewHandler(configuration.Settings{}, synchronizer, eventQueue) - return settings, eventQueue, synchronizer, handler, nil + return synchronizer, handler } diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index a07ef12..9cc9a16 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -6,6 +6,7 @@ package observation import ( + "context" "errors" "fmt" "time" @@ -17,6 +18,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" ) @@ -33,26 +35,31 @@ type Watcher struct { // informer is the informer used to watch for changes to Kubernetes resources informer cache.SharedIndexInformer + k8sClient kubernetes.Interface + // settings is the configuration settings - settings *configuration.Settings + settings configuration.Settings } // NewWatcher creates a new Watcher -func NewWatcher(settings *configuration.Settings, handler HandlerInterface) (*Watcher, error) { +func NewWatcher( + settings configuration.Settings, handler HandlerInterface, k8sClient kubernetes.Interface, +) (*Watcher, error) { return &Watcher{ - handler: handler, - settings: settings, + handler: handler, + settings: settings, + k8sClient: k8sClient, }, nil } // Initialize initializes the Watcher, must be called before Watch -func (w *Watcher) Initialize() error { +func (w *Watcher) Initialize(ctx context.Context) error { logrus.Debug("Watcher::Initialize") var err error w.informer = w.buildInformer() - err = w.initializeEventListeners() + err = w.initializeEventListeners(ctx) if err != nil { return fmt.Errorf(`initialization error: %w`, err) } @@ -62,7 +69,7 @@ func (w *Watcher) Initialize() error { // Watch starts the process of watching for changes to Kubernetes resources. // Initialize must be called before Watch. -func (w *Watcher) Watch() error { +func (w *Watcher) Watch(ctx context.Context) error { logrus.Debug("Watcher::Watch") if w.informer == nil { @@ -72,17 +79,17 @@ func (w *Watcher) Watch() error { defer utilruntime.HandleCrash() defer w.handler.ShutDown() - go w.informer.Run(w.settings.Context.Done()) + go w.informer.Run(ctx.Done()) if !cache.WaitForNamedCacheSync( w.settings.Handler.WorkQueueSettings.Name, - w.settings.Context.Done(), + ctx.Done(), w.informer.HasSynced, ) { return fmt.Errorf(`error occurred waiting for the cache to sync`) } - <-w.settings.Context.Done() + <-ctx.Done() return nil } @@ -98,7 +105,7 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { // buildEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. -func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { +func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) { logrus.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { service := obj.(*v1.Service) @@ -106,7 +113,7 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { return } - nodeIps, err := w.retrieveNodeIps() + nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return @@ -120,7 +127,7 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { // buildEventHandlerForDelete creates a function that is used as an event handler // for the informer when Delete events are raised. -func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { +func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface{}) { logrus.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { service := obj.(*v1.Service) @@ -128,7 +135,7 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { return } - nodeIps, err := w.retrieveNodeIps() + nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return @@ -142,7 +149,7 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { // buildEventHandlerForUpdate creates a function that is used as an event handler // for the informer when Update events are raised. -func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { +func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface{}, interface{}) { logrus.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API @@ -151,7 +158,7 @@ func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { return } - nodeIps, err := w.retrieveNodeIps() + nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { logrus.Errorf(`error occurred retrieving node ips: %v`, err) return @@ -168,7 +175,7 @@ func (w *Watcher) buildInformer() cache.SharedIndexInformer { logrus.Debug("Watcher::buildInformer") factory := informers.NewSharedInformerFactoryWithOptions( - w.settings.K8sClient, w.settings.Watcher.ResyncPeriod, + w.k8sClient, w.settings.Watcher.ResyncPeriod, ) informer := factory.Core().V1().Services().Informer() @@ -176,14 +183,14 @@ func (w *Watcher) buildInformer() cache.SharedIndexInformer { } // initializeEventListeners initializes the event listeners for the informer. -func (w *Watcher) initializeEventListeners() error { +func (w *Watcher) initializeEventListeners(ctx context.Context) error { logrus.Debug("Watcher::initializeEventListeners") var err error handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: w.buildEventHandlerForAdd(), - DeleteFunc: w.buildEventHandlerForDelete(), - UpdateFunc: w.buildEventHandlerForUpdate(), + AddFunc: w.buildEventHandlerForAdd(ctx), + DeleteFunc: w.buildEventHandlerForDelete(ctx), + UpdateFunc: w.buildEventHandlerForUpdate(ctx), } w.eventHandlerRegistration, err = w.informer.AddEventHandler(handlers) @@ -196,13 +203,13 @@ func (w *Watcher) initializeEventListeners() error { // notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is // because the master node may or may not be a worker node and thus may not be able to route traffic. -func (w *Watcher) retrieveNodeIps() ([]string, error) { +func (w *Watcher) retrieveNodeIps(ctx context.Context) ([]string, error) { started := time.Now() logrus.Debug("Watcher::retrieveNodeIps") var nodeIps []string - nodes, err := w.settings.K8sClient.CoreV1().Nodes().List(w.settings.Context, metav1.ListOptions{}) + nodes, err := w.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { logrus.Errorf(`error occurred retrieving the list of nodes: %v`, err) return nil, err diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 2a6d94b..f8de849 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -17,15 +17,14 @@ import ( func TestWatcher_MustInitialize(t *testing.T) { t.Parallel() watcher, _ := buildWatcher() - if err := watcher.Watch(); err == nil { + if err := watcher.Watch(context.Background()); err == nil { t.Errorf("Expected error, got %s", err) } } func buildWatcher() (*Watcher, error) { k8sClient := &kubernetes.Clientset{} - settings, _ := configuration.NewSettings(context.Background(), k8sClient) handler := &mocks.MockHandler{} - return NewWatcher(settings, handler) + return NewWatcher(configuration.Settings{}, handler, k8sClient) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 5280726..eadfbd8 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -39,12 +39,12 @@ type Interface interface { // See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { eventQueue workqueue.RateLimitingInterface - settings *configuration.Settings + settings configuration.Settings } // NewSynchronizer creates a new Synchronizer. func NewSynchronizer( - settings *configuration.Settings, + settings configuration.Settings, eventQueue workqueue.RateLimitingInterface, ) (*Synchronizer, error) { synchronizer := Synchronizer{ diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index 3634513..d1710b2 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -6,9 +6,9 @@ package synchronization import ( - "context" "fmt" "testing" + "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" @@ -17,14 +17,10 @@ import ( func TestSynchronizer_NewSynchronizer(t *testing.T) { t.Parallel() - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(configuration.Settings{}, rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -44,13 +40,10 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { UpstreamName: "", UpstreamServers: nil, } - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } + rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings(), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -72,14 +65,10 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 1 events := buildEvents(1) - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - settings.NginxPlusHosts = []string{"https://localhost:8080"} + rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings("https://localhost:8080"), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -99,18 +88,15 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { t.Parallel() const expectedEventCount = 1 events := buildEvents(1) - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - settings.NginxPlusHosts = []string{ + hosts := []string{ "https://localhost:8080", "https://localhost:8081", "https://localhost:8082", } + rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings(hosts...), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -130,13 +116,9 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 events := buildEvents(4) - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings(), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -158,14 +140,9 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 4 events := buildEvents(4) - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - settings.NginxPlusHosts = []string{"https://localhost:8080"} rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(settings, rateLimiter) + synchronizer, err := NewSynchronizer(defaultSettings("https://localhost:8080"), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -186,18 +163,16 @@ func TestSynchronizer_AddEventsManyHosts(t *testing.T) { const eventCount = 4 events := buildEvents(eventCount) rateLimiter := &mocks.MockRateLimiter{} - settings, err := configuration.NewSettings(context.Background(), nil) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - settings.NginxPlusHosts = []string{ + + hosts := []string{ "https://localhost:8080", "https://localhost:8081", "https://localhost:8082", } - expectedEventCount := eventCount * len(settings.NginxPlusHosts) - synchronizer, err := NewSynchronizer(settings, rateLimiter) + expectedEventCount := eventCount * len(hosts) + + synchronizer, err := NewSynchronizer(defaultSettings(hosts...), rateLimiter) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -226,3 +201,20 @@ func buildEvents(count int) core.ServerUpdateEvents { } return events } + +func defaultSettings(nginxHosts ...string) configuration.Settings { + return configuration.Settings{ + NginxPlusHosts: nginxHosts, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + } +} From 32cb7589da1309d3c61aec639f607de2c5fc65af Mon Sep 17 00:00:00 2001 From: Nathan Bird Date: Mon, 30 Sep 2024 15:19:01 -0400 Subject: [PATCH 044/136] Get rid of helm chart cruft --- charts/nlk/Chart.yaml | 2 +- charts/nlk/values.yaml | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index 1f986a3..bea170b 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -7,8 +7,8 @@ home: https://github.com/nginxinc/nginx-loadbalancer-kubernetes icon: https://raw.githubusercontent.com/nginxinc/nginx-loadbalancer-kubernetes/main/nlk-logo.svg keywords: - nginx +- nginxaas - loadbalancer -- ingress kubeVersion: '>= 1.22.0-0' maintainers: - name: "@ciroque" diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 11d082b..b32fb4c 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -42,22 +42,6 @@ nlk: type: ClusterIP port: 80 - ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - resources: requests: cpu: 100m @@ -66,13 +50,6 @@ nlk: # cpu: 100m # memory: 128Mi - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 3 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - # Additional volumes on the output Deployment definition. volumes: [] # - name: foo From ece843f10bf63d4bd38935384d4ceb6bbcfe16f0 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 30 Sep 2024 18:29:09 -0700 Subject: [PATCH 045/136] NLB-5678: Template the namespace We should respect the user supplied release namespace in the helm chart instead of hardcoding it to `nlk`. --- charts/nlk/templates/clusterrolebinding.yaml | 2 +- charts/nlk/templates/nlk-configmap.yaml | 2 +- charts/nlk/templates/nlk-deployment.yaml | 2 +- charts/nlk/templates/nlk-secret.yaml | 2 +- charts/nlk/templates/nlk-serviceaccount.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/nlk/templates/clusterrolebinding.yaml b/charts/nlk/templates/clusterrolebinding.yaml index 0ccd455..8503a24 100644 --- a/charts/nlk/templates/clusterrolebinding.yaml +++ b/charts/nlk/templates/clusterrolebinding.yaml @@ -6,7 +6,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: {{ .Release.Namespace }}-{{ include "nlk.fullname" . }} diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index ab8977f..8cd4d66 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: nlk-config - namespace: nlk + namespace: {{ .Release.Namespace }} data: config.yaml: | {{- if .Values.nlk.config.entries.hosts }} diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 3b5348e..35de7f1 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} labels: app: nlk spec: diff --git a/charts/nlk/templates/nlk-secret.yaml b/charts/nlk/templates/nlk-secret.yaml index ff7d7ff..cb96486 100644 --- a/charts/nlk/templates/nlk-secret.yaml +++ b/charts/nlk/templates/nlk-secret.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} annotations: kubernetes.io/service-account.name: {{ include "nlk.fullname" . }} type: kubernetes.io/service-account-token diff --git a/charts/nlk/templates/nlk-serviceaccount.yaml b/charts/nlk/templates/nlk-serviceaccount.yaml index 5bdca4f..d2cd8e4 100644 --- a/charts/nlk/templates/nlk-serviceaccount.yaml +++ b/charts/nlk/templates/nlk-serviceaccount.yaml @@ -3,5 +3,5 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "nlk.fullname" . }} - namespace: nlk + namespace: {{ .Release.Namespace }} {{- end }} From df989bf3f287e08f6c631915bdbd15bea7a8dc4b Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 27 Sep 2024 16:25:57 -0700 Subject: [PATCH 046/136] NLB-5666: Inject dataplane API key via Helm THe operator needs the dataplane API Key to authenticate with the deployment dataplane endpoint. This commit adds the abailitiy for a user to supply a key via Helm, create a secret on their behalf, and then mounts it into the pod. --- charts/nlk/templates/_helpers.tpl | 4 ++++ charts/nlk/templates/dataplaneApiKey.yaml | 8 ++++++++ charts/nlk/templates/nlk-deployment.yaml | 6 ++++++ charts/nlk/values.yaml | 2 ++ version | 2 +- 5 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 charts/nlk/templates/dataplaneApiKey.yaml diff --git a/charts/nlk/templates/_helpers.tpl b/charts/nlk/templates/_helpers.tpl index 17a6405..119c164 100644 --- a/charts/nlk/templates/_helpers.tpl +++ b/charts/nlk/templates/_helpers.tpl @@ -48,6 +48,10 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} +{{- define "nlk.apikeyname" -}} +{{- printf "%s-nginxaas-api-key" (include "nlk.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} + {{/* Common labels */}} diff --git a/charts/nlk/templates/dataplaneApiKey.yaml b/charts/nlk/templates/dataplaneApiKey.yaml new file mode 100644 index 0000000..11f76f8 --- /dev/null +++ b/charts/nlk/templates/dataplaneApiKey.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include nlk.apikeyname . }} + namespace: {{ .Release.Namespace }} +type: Opaque +data: + nginxaasApiKey: {{ .Values.nlk.dataplaneApiKey | toString | b64enc }} diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 35de7f1..eb00b3a 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -45,6 +45,12 @@ spec: initialDelaySeconds: {{ .Values.nlk.readyStatus.initialDelaySeconds }} periodSeconds: {{ .Values.nlk.readyStatus.periodSeconds }} {{- end }} + env: + - name: NGINXAAS_DATAPLANE_API_KEY + valueFrom: + secretKeyRef: + name: {{ include nlk.apikeyname . }} + key: nginxaasApiKey volumeMounts: - name: config mountPath: /etc/nginxaas-operator diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index b32fb4c..33889c3 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -79,6 +79,8 @@ nlk: ca-certificate: "" client-certificate: "" + dataplaneApiKey: "" + logLevel: "warn" containerPort: diff --git a/version b/version index 9faa1b7..0ea3a94 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.1.5 +0.2.0 From 1f84f62f3a39123c55506740e9d3a7ccd2e9cb81 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 00:29:39 -0700 Subject: [PATCH 047/136] Cleanup helm chart values These are visible values that do not do anything on the chart itself for now. Cleaning these up and we can add them as we make progress on nlk. --- charts/nlk/Chart.yaml | 4 +--- charts/nlk/values.yaml | 47 +----------------------------------------- 2 files changed, 2 insertions(+), 49 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index bea170b..64f735a 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,10 +1,8 @@ --- apiVersion: v2 appVersion: 0.1.0 -description: NGINX LoadBalancer for Kubernetes +description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-operator -home: https://github.com/nginxinc/nginx-loadbalancer-kubernetes -icon: https://raw.githubusercontent.com/nginxinc/nginx-loadbalancer-kubernetes/main/nlk-logo.svg keywords: - nginx - nginxaas diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 33889c3..4e7e71d 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -24,55 +24,10 @@ nlk: # Annotations to add to the service account annotations: {} - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - resources: - requests: - cpu: 100m - memory: 128Mi - # limits: - # cpu: 100m - # memory: 128Mi - - # Additional volumes on the output Deployment definition. - volumes: [] - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - nodeSelector: {} - - tolerations: [] - - affinity: {} - config: entries: hosts: - "http://10.1.1.4:9000/api,http://10.1.1.5:9000/api" + "" defaultTLS: tls-mode: "no-tls" From e7e629a0c81f40b78ac976724dede48079aa4dbe Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 11:24:40 -0700 Subject: [PATCH 048/136] Quote the helm function --- charts/nlk/templates/dataplaneApiKey.yaml | 2 +- charts/nlk/templates/nlk-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/nlk/templates/dataplaneApiKey.yaml b/charts/nlk/templates/dataplaneApiKey.yaml index 11f76f8..2051138 100644 --- a/charts/nlk/templates/dataplaneApiKey.yaml +++ b/charts/nlk/templates/dataplaneApiKey.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Secret metadata: - name: {{ include nlk.apikeyname . }} + name: {{ include "nlk.apikeyname" . }} namespace: {{ .Release.Namespace }} type: Opaque data: diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index eb00b3a..bc12f5c 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -49,7 +49,7 @@ spec: - name: NGINXAAS_DATAPLANE_API_KEY valueFrom: secretKeyRef: - name: {{ include nlk.apikeyname . }} + name: {{ include "nlk.apikeyname" . }} key: nginxaasApiKey volumeMounts: - name: config From 5b306543a3ca20954c30a25c71aa19123a5c53fb Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 11:58:04 -0700 Subject: [PATCH 049/136] Default the API Key value in helm The default is set as not doing so causes helm to produce an empty secret cause the pod to fail the secret mount. This was missed on original testing as I did not realize that a user could skip specifying the secret as well. --- charts/nlk/values.yaml | 3 ++- version | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 4e7e71d..19713f5 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -34,7 +34,8 @@ nlk: ca-certificate: "" client-certificate: "" - dataplaneApiKey: "" + # Override with your own NGINXaaS dataplane API Key. + dataplaneApiKey: "test" logLevel: "warn" diff --git a/version b/version index 0ea3a94..0c62199 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.2.0 +0.2.1 From a6b8e3b5ba7f44fdea05a8875bed24ba9a1d757a Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 18:41:15 -0700 Subject: [PATCH 050/136] Rename nginxaas-operator to nginxaas-loadbalancer-kubernetes --- .gitignore | 2 +- Dockerfile | 6 +++--- charts/nlk/Chart.yaml | 2 +- charts/nlk/templates/nlk-deployment.yaml | 2 +- charts/nlk/values.yaml | 4 ++-- cmd/nginx-loadbalancer-kubernetes/main.go | 2 +- scripts/build.sh | 2 +- scripts/docker.sh | 2 +- scripts/publish-helm.sh | 2 +- scripts/release.sh | 8 ++++---- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index f434c43..e69f538 100644 --- a/.gitignore +++ b/.gitignore @@ -88,4 +88,4 @@ results .go-build -nginxaas-operator-* +nginxaas-loadbalancer-kubernetes-* diff --git a/Dockerfile b/Dockerfile index d80281c..e53aef4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,6 @@ COPY docker-user /etc/passwd USER 101 COPY --from=base-certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -FROM base as nginxaas-operator -ENTRYPOINT ["/nginxaas-operator"] -COPY build/nginxaas-operator / +FROM base as nginxaas-loadbalancer-kubernetes +ENTRYPOINT ["/nginxaas-loadbalancer-kubernetes"] +COPY build/nginxaas-loadbalancer-kubernetes / diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index 64f735a..5c8d97c 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 0.1.0 description: NGINXaaS LoadBalancer for Kubernetes -name: nginxaas-operator +name: nginxaas-loadbalancer-kubernetes keywords: - nginx - nginxaas diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index bc12f5c..826e251 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -53,7 +53,7 @@ spec: key: nginxaasApiKey volumeMounts: - name: config - mountPath: /etc/nginxaas-operator + mountPath: /etc/nginxaas-loadbalancer-kubernetes serviceAccountName: {{ include "nlk.fullname" . }} volumes: - name: config diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 19713f5..3004231 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -1,5 +1,5 @@ nlk: - name: nginxaas-operator + name: nginxaas-loadbalancer-kubernetes kind: deployment @@ -7,7 +7,7 @@ nlk: image: registry: registry-1.docker.io - repository: nginx/nginxaas-operator + repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. tag: release-0.1.0 diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index f8473c3..a9f6cd9 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -35,7 +35,7 @@ func run() error { return fmt.Errorf(`error building a Kubernetes client: %w`, err) } - settings, err := configuration.Read("config.yaml", "/etc/nginxaas-operator") + settings, err := configuration.Read("config.yaml", "/etc/nginxaas-loadbalancer-kubernetes") if err != nil { return fmt.Errorf(`error occurred accessing configuration: %w`, err) } diff --git a/scripts/build.sh b/scripts/build.sh index aafb418..fb1a634 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -36,5 +36,5 @@ ldflags=( go build \ -v -tags "release osusergo" \ -ldflags "${ldflags[*]}" \ - -o "${BUILD_DIR}/nginxaas-operator" \ + -o "${BUILD_DIR}/nginxaas-loadbalancer-kubernetes" \ "$pkg_path" diff --git a/scripts/docker.sh b/scripts/docker.sh index af473f1..6208b79 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -69,7 +69,7 @@ parse_args() { } # MAIN -image="nginxaas-operator" +image="nginxaas-loadbalancer-kubernetes" parse_args "$@" init_ci_vars diff --git a/scripts/publish-helm.sh b/scripts/publish-helm.sh index 8abecc5..71155d9 100755 --- a/scripts/publish-helm.sh +++ b/scripts/publish-helm.sh @@ -5,7 +5,7 @@ set -eo pipefail ROOT_DIR=$(git rev-parse --show-toplevel) publish_helm() { - pkg="nginxaas-operator-${version}.tgz" + pkg="nginxaas-loadbalancer-kubernetes-${version}.tgz" helm package --version "${version}" --app-version "${version}" charts/nlk helm push "${pkg}" "${repo}" } diff --git a/scripts/release.sh b/scripts/release.sh index aa942c8..6de2182 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,10 +3,10 @@ set -eo pipefail docker-image() { - SRC_PATH="nginx-azure-lb/nginxaas-operator/nginxaas-operator" + SRC_PATH="nginx-azure-lb/nginxaas-loadbalancer-kubernetes/nginxaas-loadbalancer-kubernetes" SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:main-${SRC_TAG}" - DST_PATH="nginx/nginxaas-operator" + DST_PATH="nginx/nginxaas-loadbalancer-kubernetes" DST_TAG="${CI_COMMIT_TAG}" DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" @@ -16,7 +16,7 @@ docker-image() { } helm-chart() { - SRC_PATH="nginx-azure-lb/nginxaas-operator/charts/main/nginxaas-operator" + SRC_PATH="nginx-azure-lbnginxaas-loadbalancer-kubernetes/charts/main/nginxaas-loadbalancer-kubernetes" SRC_TAG="0.1.0" SRC_CHART="oci://${SRC_REGISTRY}/${SRC_PATH}" DST_PATH="nginxcharts" @@ -24,7 +24,7 @@ helm-chart() { DST_CHART="oci://${DST_REGISTRY}/${DST_PATH}" helm pull "${SRC_CHART}" --version "${SRC_TAG}" - helm push nginxaas-operator-${DST_TAG}.tgz "${DST_CHART}" + helm push nginxaas-loadbalancer-kubernetes-${DST_TAG}.tgz "${DST_CHART}" } From d1b5186043b8e2885164ab3b2b33554c60ab9177 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 2 Oct 2024 13:09:10 -0700 Subject: [PATCH 051/136] Fix helm repo path --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index 6de2182..730dfcb 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -16,7 +16,7 @@ docker-image() { } helm-chart() { - SRC_PATH="nginx-azure-lbnginxaas-loadbalancer-kubernetes/charts/main/nginxaas-loadbalancer-kubernetes" + SRC_PATH="nginx-azure-lb/nginxaas-loadbalancer-kubernetes/charts/main/nginxaas-loadbalancer-kubernetes" SRC_TAG="0.1.0" SRC_CHART="oci://${SRC_REGISTRY}/${SRC_PATH}" DST_PATH="nginxcharts" From 148dcc80ba597ef279ec26f9a058d309215a3084 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 2 Oct 2024 13:29:52 -0700 Subject: [PATCH 052/136] Version charts and images together We started doing this but I missed out fixing up the helm chart version in the release CI. In addition to versioning these together, I am also proposing that instead of cutting tags like `release-x.y.z`, we should use `v1.0.0`. --- scripts/release.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 730dfcb..6ea4dd0 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -4,10 +4,8 @@ set -eo pipefail docker-image() { SRC_PATH="nginx-azure-lb/nginxaas-loadbalancer-kubernetes/nginxaas-loadbalancer-kubernetes" - SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "-") SRC_IMG="${SRC_REGISTRY}/${SRC_PATH}:main-${SRC_TAG}" DST_PATH="nginx/nginxaas-loadbalancer-kubernetes" - DST_TAG="${CI_COMMIT_TAG}" DST_IMG="${DST_REGISTRY}/${DST_PATH}:${DST_TAG}" docker pull "${SRC_IMG}" @@ -17,10 +15,8 @@ docker-image() { helm-chart() { SRC_PATH="nginx-azure-lb/nginxaas-loadbalancer-kubernetes/charts/main/nginxaas-loadbalancer-kubernetes" - SRC_TAG="0.1.0" SRC_CHART="oci://${SRC_REGISTRY}/${SRC_PATH}" DST_PATH="nginxcharts" - DST_TAG="0.1.0" DST_CHART="oci://${DST_REGISTRY}/${DST_PATH}" helm pull "${SRC_CHART}" --version "${SRC_TAG}" @@ -51,6 +47,8 @@ set_docker_common() { devops.docker.login # Login to Dockerhub. docker login --username "${DOCKERHUB_USERNAME}" --password "${DOCKERHUB_PASSWORD}" "${DST_REGISTRY}" + SRC_TAG=$(echo "${CI_COMMIT_TAG}" | cut -f 2 -d "v") + DST_TAG="${SRC_TAG}" } parse_args() { From 7250a55326b7dc04fd04830f8ff3885a63051124 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 2 Oct 2024 13:32:30 -0700 Subject: [PATCH 053/136] Fix up helm and docker versions This just brings them up to the latest. --- charts/nlk/Chart.yaml | 4 ++-- charts/nlk/values.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index 5c8d97c..f7f4c9c 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,6 +1,6 @@ --- apiVersion: v2 -appVersion: 0.1.0 +appVersion: 0.2.1 description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-loadbalancer-kubernetes keywords: @@ -14,4 +14,4 @@ maintainers: - name: "@abdennour" type: application -version: 0.1.0 +version: 0.2.1 diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 3004231..14a2b4f 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -10,7 +10,7 @@ nlk: repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: release-0.1.0 + tag: 0.2.1 imagePullSecrets: [] nameOverride: "" From d56d38befe37a82793eaf9a872b4e9a90674ad17 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 2 Oct 2024 13:52:58 -0700 Subject: [PATCH 054/136] Fix up release regex to allow a release --- .gitlab-ci.yml | 2 +- scripts/release.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3d94f4..dabd2b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -183,7 +183,7 @@ checkmarx-scan: extends: - .devops-docker-cicd rules: - - if: '$CI_COMMIT_TAG =~ /^release-[\d]+\.[\d]+\.[\d]+/' + - if: '$CI_COMMIT_TAG =~ /^v[\d]+\.[\d]+\.[\d]+/' dockerhub-image-release: extends: diff --git a/scripts/release.sh b/scripts/release.sh index 6ea4dd0..f8ed556 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -72,7 +72,7 @@ main() { echo "This script is meant to be run in the CI." exit 1 fi - pttn="^release-[0-9]+\.[0-9]+\.[0-9]+" + pttn="^v[0-9]+\.[0-9]+\.[0-9]+" if ! [[ "${CI_COMMIT_TAG}" =~ $pttn ]]; then echo "CI_COMMIT_TAG needs to be set to valid semver format." exit 1 From ea9d9d19843968f80d95bfb181595d9f961789b2 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 1 Oct 2024 18:08:22 -0700 Subject: [PATCH 055/136] Fixup nlk configmap name We should really be templating the configmap name rather than hardcoding it (similar to what we did for secrets). --- charts/nlk/templates/_helpers.tpl | 6 +----- charts/nlk/templates/nlk-configmap.yaml | 2 +- charts/nlk/templates/nlk-deployment.yaml | 2 +- version | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/charts/nlk/templates/_helpers.tpl b/charts/nlk/templates/_helpers.tpl index 119c164..3f62fe9 100644 --- a/charts/nlk/templates/_helpers.tpl +++ b/charts/nlk/templates/_helpers.tpl @@ -80,11 +80,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} Expand the name of the configmap. */}} {{- define "nlk.configName" -}} -{{- if .Values.nlk.customConfigMap -}} -{{ .Values.nlk.customConfigMap }} -{{- else -}} -{{- default (include "nlk.fullname" .) .Values.nlk.config.name -}} -{{- end -}} +{{- printf "%s-nlk-config" (include "nlk.fullname" .) | trunc 63 | trimSuffix "-" }} {{- end -}} {{/* diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index 8cd4d66..78d4140 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: nlk-config + name: {{ include "nlk.configName" . }} namespace: {{ .Release.Namespace }} data: config.yaml: | diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 826e251..e49006f 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -58,4 +58,4 @@ spec: volumes: - name: config configMap: - name: nlk-config + name: {{ include "nlk.configName" . }} diff --git a/version b/version index 0c62199..ee1372d 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.2.1 +0.2.2 From 1b921702e7e26a087bb3dea6fee39bb9e0d3950e Mon Sep 17 00:00:00 2001 From: Nathan Bird Date: Tue, 1 Oct 2024 17:15:23 -0400 Subject: [PATCH 056/136] Remove configmap and secret permissions This is not a permission we need right now. If we merge back to NLK upstream we might want to _optionally_ restore this if the user needs to read certificate information. --- charts/nlk/Chart.yaml | 4 ++-- charts/nlk/templates/clusterrole.yaml | 2 -- charts/nlk/values.yaml | 2 +- version | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index f7f4c9c..a7e619a 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,6 +1,6 @@ --- apiVersion: v2 -appVersion: 0.2.1 +appVersion: 0.3.0 description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-loadbalancer-kubernetes keywords: @@ -14,4 +14,4 @@ maintainers: - name: "@abdennour" type: application -version: 0.2.1 +version: 0.3.0 diff --git a/charts/nlk/templates/clusterrole.yaml b/charts/nlk/templates/clusterrole.yaml index 4164475..c652c4c 100644 --- a/charts/nlk/templates/clusterrole.yaml +++ b/charts/nlk/templates/clusterrole.yaml @@ -7,9 +7,7 @@ rules: - apiGroups: - "" resources: - - configmaps - nodes - - secrets - services verbs: - get diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 14a2b4f..5fc1c0d 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -10,7 +10,7 @@ nlk: repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: 0.2.1 + tag: 0.3.0 imagePullSecrets: [] nameOverride: "" diff --git a/version b/version index ee1372d..0d91a54 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.2.2 +0.3.0 From 7ac864bb6b74a930882934c27ff129eacf920353 Mon Sep 17 00:00:00 2001 From: Nathan Bird Date: Tue, 1 Oct 2024 17:18:34 -0400 Subject: [PATCH 057/136] Put config values in config key - group all the config vars together - make naming more consistent - update doc strings -- use ## to describe, # to show the key:value - remove references to non-supported ca certificate information - add nlk.config.serviceAnnotationMatch Documentation example ## trace,debug,info,warn,error,fatal,panic # logLevel: "warn" --- charts/nlk/Chart.yaml | 4 ++-- charts/nlk/templates/nlk-configmap.yaml | 15 +++++++----- charts/nlk/values.yaml | 32 +++++++++++++------------ version | 2 +- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index a7e619a..4e1860f 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,6 +1,6 @@ --- apiVersion: v2 -appVersion: 0.3.0 +appVersion: 0.4.0 description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-loadbalancer-kubernetes keywords: @@ -14,4 +14,4 @@ maintainers: - name: "@abdennour" type: application -version: 0.3.0 +version: 0.4.0 diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index 78d4140..38475a9 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -5,10 +5,13 @@ metadata: namespace: {{ .Release.Namespace }} data: config.yaml: | -{{- if .Values.nlk.config.entries.hosts }} - nginx-hosts: "{{ .Values.nlk.config.entries.hosts }}" +{{- with .Values.nlk.config.logLevel }} + log-level: "{{ . }}" +{{- end }} +{{- with .Values.nlk.config.nginxHosts }} + nginx-hosts: "{{ . }}" +{{- end }} + tls-mode: "{{ .Values.nlk.config.tls.mode }}" +{{- with .Values.nlk.config.serviceAnnotationMatch }} + service-annotation-match: "{{ . }}" {{- end }} - tls-mode: "{{ index .Values.nlk.defaultTLS "tls-mode" }}" - ca-certificate: "{{ index .Values.nlk.defaultTLS "ca-certificate" }}" - client-certificate: "{{ index .Values.nlk.defaultTLS "client-certificate" }}" - log-level: "{{ .Values.nlk.logLevel }}" diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 5fc1c0d..75f6b54 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -9,35 +9,37 @@ nlk: registry: registry-1.docker.io repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - tag: 0.3.0 + ## Overrides the image tag whose default is the chart appVersion. + tag: 0.4.0 imagePullSecrets: [] nameOverride: "" fullnameOverride: "" serviceAccount: - # Specifies whether a service account should be created + ## Specifies whether a service account should be created create: true - # Automatically mount a ServiceAccount's API credentials? + ## Automatically mount a ServiceAccount's API credentials? automount: true - # Annotations to add to the service account + ## Annotations to add to the service account annotations: {} config: - entries: - hosts: - "" + ## trace,debug,info,warn,error,fatal,panic + # logLevel: "warn" - defaultTLS: - tls-mode: "no-tls" - ca-certificate: "" - client-certificate: "" + ## the nginx hosts (comma-separated) to send upstream updates to + nginxHosts: "" - # Override with your own NGINXaaS dataplane API Key. - dataplaneApiKey: "test" + ## Sets the annotation value that NLK is looking for to watch a Service + # serviceAnnotationMatch: nginxaas + + tls: + ## can also be set to "no-tls" to disable server cert verification + mode: "ca-tls" - logLevel: "warn" + ## Override with your own NGINXaaS dataplane API Key. + dataplaneApiKey: "test" containerPort: http: 51031 diff --git a/version b/version index 0d91a54..1d0ba9e 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.3.0 +0.4.0 From 63c82a70f52fa9c7756d2d21133c287073b8931c Mon Sep 17 00:00:00 2001 From: sarna Date: Sun, 6 Oct 2024 22:57:51 -0700 Subject: [PATCH 058/136] Remove unused YAMLs We have officially transitions towards using Helm to install NLK. With that, we should also look at auto-generating these YAMLs via helm and not hardcode them into the repository. Removing them as they also add additional grep confusion while scanning through the repo. --- deployments/checks/sample-ingress-mod.yaml | 18 ---------- deployments/checks/sample-ingress.yaml | 18 ---------- deployments/deployment/configmap.yaml | 11 ------- deployments/deployment/deployment.yaml | 38 ---------------------- deployments/deployment/namespace.yaml | 6 ---- deployments/rbac/apply.sh | 12 ------- deployments/rbac/clusterrole.yaml | 10 ------ deployments/rbac/clusterrolebinding.yaml | 13 -------- deployments/rbac/secret.yaml | 8 ----- deployments/rbac/serviceaccount.yaml | 5 --- deployments/rbac/unapply.sh | 8 ----- 11 files changed, 147 deletions(-) delete mode 100644 deployments/checks/sample-ingress-mod.yaml delete mode 100644 deployments/checks/sample-ingress.yaml delete mode 100644 deployments/deployment/configmap.yaml delete mode 100644 deployments/deployment/deployment.yaml delete mode 100644 deployments/deployment/namespace.yaml delete mode 100755 deployments/rbac/apply.sh delete mode 100644 deployments/rbac/clusterrole.yaml delete mode 100644 deployments/rbac/clusterrolebinding.yaml delete mode 100644 deployments/rbac/secret.yaml delete mode 100644 deployments/rbac/serviceaccount.yaml delete mode 100755 deployments/rbac/unapply.sh diff --git a/deployments/checks/sample-ingress-mod.yaml b/deployments/checks/sample-ingress-mod.yaml deleted file mode 100644 index cd79305..0000000 --- a/deployments/checks/sample-ingress-mod.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: example-ingress - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /$1 -spec: - rules: - - host: hello-world.net - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: web - port: - number: 8080 \ No newline at end of file diff --git a/deployments/checks/sample-ingress.yaml b/deployments/checks/sample-ingress.yaml deleted file mode 100644 index 7ff7fc5..0000000 --- a/deployments/checks/sample-ingress.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: example-ingress - annotations: - nginx.ingress.kubernetes.io/rewrite-target: /$1 -spec: - rules: - - host: hello-world.com - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: web - port: - number: 8080 \ No newline at end of file diff --git a/deployments/deployment/configmap.yaml b/deployments/deployment/configmap.yaml deleted file mode 100644 index fd30dbe..0000000 --- a/deployments/deployment/configmap.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -data: - nginx-hosts: "https://10.0.0.1:9000/api" - tls-mode: "no-tls" - ca-certificate: "" - client-certificate: "" - log-level: "warn" -metadata: - name: nlk-config - namespace: nlk diff --git a/deployments/deployment/deployment.yaml b/deployments/deployment/deployment.yaml deleted file mode 100644 index 4c871c2..0000000 --- a/deployments/deployment/deployment.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nlk-deployment - namespace: nlk - labels: - app: nlk -spec: - replicas: 1 - selector: - matchLabels: - app: nlk - template: - metadata: - labels: - app: nlk - spec: - containers: - - name: nginx-loadbalancer-kubernetes - image: ghcr.io/nginxinc/nginx-loadbalancer-kubernetes:latest - imagePullPolicy: Always - ports: - - name: http - containerPort: 51031 - protocol: TCP - livenessProbe: - httpGet: - path: /livez - port: 51031 - initialDelaySeconds: 5 - periodSeconds: 2 - readinessProbe: - httpGet: - path: /readyz - port: 51031 - initialDelaySeconds: 5 - periodSeconds: 2 - serviceAccountName: nginx-loadbalancer-kubernetes diff --git a/deployments/deployment/namespace.yaml b/deployments/deployment/namespace.yaml deleted file mode 100644 index 8f9e382..0000000 --- a/deployments/deployment/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: nlk - labels: - name: nlk diff --git a/deployments/rbac/apply.sh b/deployments/rbac/apply.sh deleted file mode 100755 index 58248da..0000000 --- a/deployments/rbac/apply.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -pushd "$(dirname "$0")" - -echo "Applying all RBAC resources..." - -kubectl apply -f serviceaccount.yaml -kubectl apply -f clusterrole.yaml -kubectl apply -f clusterrolebinding.yaml -kubectl apply -f secret.yaml - -popd diff --git a/deployments/rbac/clusterrole.yaml b/deployments/rbac/clusterrole.yaml deleted file mode 100644 index c50bed8..0000000 --- a/deployments/rbac/clusterrole.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: resource-get-watch-list - namespace: nlk -rules: - - apiGroups: - - "" - resources: ["services", "nodes", "configmaps", "secrets"] - verbs: ["get", "watch", "list"] diff --git a/deployments/rbac/clusterrolebinding.yaml b/deployments/rbac/clusterrolebinding.yaml deleted file mode 100644 index d48ffb8..0000000 --- a/deployments/rbac/clusterrolebinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: "nginx-loadbalancer-kubernetes:resource-get-watch-list" - namespace: nlk -subjects: - - kind: ServiceAccount - name: nginx-loadbalancer-kubernetes - namespace: nlk -roleRef: - kind: ClusterRole - name: resource-get-watch-list - apiGroup: rbac.authorization.k8s.io diff --git a/deployments/rbac/secret.yaml b/deployments/rbac/secret.yaml deleted file mode 100644 index 71576bf..0000000 --- a/deployments/rbac/secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: nginx-loadbalancer-kubernetes-secret - namespace: nlk - annotations: - kubernetes.io/service-account.name: nginx-loadbalancer-kubernetes -type: kubernetes.io/service-account-token diff --git a/deployments/rbac/serviceaccount.yaml b/deployments/rbac/serviceaccount.yaml deleted file mode 100644 index 76f238c..0000000 --- a/deployments/rbac/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-loadbalancer-kubernetes - namespace: nlk diff --git a/deployments/rbac/unapply.sh b/deployments/rbac/unapply.sh deleted file mode 100755 index f29f90d..0000000 --- a/deployments/rbac/unapply.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -echo "Unapplying all RBAC resources..." - -kubectl delete -f serviceaccount.yaml -kubectl delete -f clusterrole.yaml -kubectl delete -f clusterrolebinding.yaml -kubectl delete -f secret.yaml From 3e98257a8862ca2c48b868d6400c126eeec83ede Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 8 Oct 2024 15:08:01 -0600 Subject: [PATCH 059/136] NLB-5679 Added checksum field to helm nlk-deployment to force restart on configmap change --- charts/nlk/templates/nlk-deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index e49006f..6270822 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -14,6 +14,8 @@ spec: metadata: labels: app: nlk + annotations: + checksum: {{ tpl (toYaml .Values.nlk) . | sha256sum }} spec: {{- with .Values.nlk.imagePullSecrets }} imagePullSecrets: From cbe6dfa4f30246db8b89f02f18b2531006200588 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 9 Oct 2024 10:34:39 -0600 Subject: [PATCH 060/136] NLB-5679 bumped to v0.4.1 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 1d0ba9e..267577d 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.4.0 +0.4.1 From 332ea4fd2be24457a8fbfd0562ddfe2b8bf30012 Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 9 Oct 2024 10:25:26 -0700 Subject: [PATCH 061/136] NLB-5432: Add Gitlab Container scanning Gitlab uses Trivy underneath to scan an image which aligns with what we should try to use anyway. The nice thing about this work is: - Native integration with Gitlab. - Gitlab keeps Vulnerability DB up to date daily so we don't have to. --- .gitlab-ci.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dabd2b2..ab24a67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,9 +12,11 @@ include: - project: "f5/nginx/tools/devops-utils" file: "/include/gitlab-sast.yml" ref: "master" + - template: Jobs/Container-Scanning.gitlab-ci.yml stages: - lint+test+build + - security scanning - release variables: @@ -177,6 +179,34 @@ checkmarx-scan: needs: [] allow_failure: true +# https://docs.gitlab.com/ee/user/application_security/container_scanning/ +container_scanning: + stage: security scanning + extends: + - .devops-docker-cicd + variables: + CS_DISABLE_LANGUAGE_VULNERABILITY_SCAN: "false" + GIT_STRATEGY: "fetch" + # CI_COMMIT_SHORT_SHA tag is being used to uniquely scan images being produced (or reproduced) on a feature branch. + CS_IMAGE: "${DEVOPS_DOCKER_URL_DEFAULT}/nginx-azure-lb/${CI_PROJECT_NAME}/nginxaas-loadbalancer-kubernetes:${CI_COMMIT_SHORT_SHA}" + # Gitlab container scanner needs the tenant ID to be set as it is using Azure auth libs underneath. + AZURE_TENANT_ID: ${ARM_TENANT_ID} + before_script: + # The Gitlab container scan image runs as a non-root user, which leads to the DevOps abstraction failing to bootstrap + # as the CI directories on the runner as seeded as "root". + - git config --global --add safe.directory "${CI_PROJECT_DIR}" + # Prerequisite toolchain to run the DevOps abstraction. + - sudo apt-get update && sudo apt-get install -y curl jq + - *import-devops-core-services + - devops.backend.docker.authenticate + - export CS_REGISTRY_USER="${DEVOPS_DOCKER_USER}" CS_REGISTRY_PASSWORD="${DEVOPS_DOCKER_PASS}" + rules: + - if: '$CI_COMMIT_TAG =~ /^v[\d]+\.[\d]+\.[\d]+/' + when: never + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - when: on_success + .release-common: stage: release image: $DEVTOOLS_IMG From 7cbd48b942ff00448469bab2915b08ecb1920e65 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 17:27:00 -0700 Subject: [PATCH 062/136] Add linting for helm chart --- .gitlab-ci.yml | 1 + Makefile | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab24a67..d471ae7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,6 +89,7 @@ lint + unit-test + build: - *golang-private - | if [ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]; then + time make helm-lint time make lint git diff --exit-code time make test diff --git a/Makefile b/Makefile index f0c88a8..e53c440 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ DOCKER_TAG ?= latest GOPRIVATE = *.f5net.com,gitlab.com/f5 export GOPRIVATE -.PHONY: default tools deps fmt lint test build build.docker publish +.PHONY: default tools deps fmt lint test build build.docker publish helm-lint default: build @@ -35,6 +35,9 @@ lint: @find . -type f -name "*.go" -exec goimports -e -w {} \+ @golangci-lint run -v ./... +helm-lint: + helm lint charts/nlk/ + test: @./scripts/test.sh From 7c69d10bbb3f76cbd045162c8c076d21c8215979 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 7 Oct 2024 17:43:56 -0700 Subject: [PATCH 063/136] NLB-5717: Add manifest for marketplace app This is the manifest describing the app on Azure marketplace. --- charts/manifest.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 charts/manifest.yaml diff --git a/charts/manifest.yaml b/charts/manifest.yaml new file mode 100644 index 0000000..9138b81 --- /dev/null +++ b/charts/manifest.yaml @@ -0,0 +1,10 @@ +applicationName: "marketplace/nginxaas-loadbalancer-kubernetes" +publisher: "F5, Inc." +description: "A component that manages NGINXaaS for Azure deployment and makes it act as Load Balancer for kubernetes workloads." +version: 0.4.0 +helmChart: "./nlk" +clusterArmTemplate: "./armTemplate.json" +uiDefinition: "./createUIDefinition.json" +registryServer: "nlbmarketplaceacrprod.azurecr.io" +extensionRegistrationParameters: + defaultScope: "namespace" From afe5c006ecef0c5f2105bb4410316e8a366aa3e8 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 17:46:43 -0700 Subject: [PATCH 064/136] NLB-5707: Create UI definition for AKS marketplace This file describes what the UX will look like for the NLK extension on the AKS marketplace. --- charts/createUIDefinition.json | 254 +++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 charts/createUIDefinition.json diff --git a/charts/createUIDefinition.json b/charts/createUIDefinition.json new file mode 100644 index 0000000..906e08e --- /dev/null +++ b/charts/createUIDefinition.json @@ -0,0 +1,254 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#", + "handler": "Microsoft.Azure.CreateUIDef", + "version": "0.1.2-preview", + "parameters": { + "config": { + "isWizard": false, + "basics": { + "location": { + "visible": "[basics('createNewCluster')]", + "resourceTypes": ["Nginx.NginxPlus/nginxDeployments"] + }, + "resourceGroup": { + "allowExisting": true + } + } + }, + "basics": [ + { + "name": "createNewCluster", + "type": "Microsoft.Common.OptionsGroup", + "label": "Create new AKS cluster", + "defaultValue": "No", + "toolTip": "Create a new AKS cluster to install the extension.", + "constraints": { + "allowedValues": [ + { + "label": "Yes", + "value": true + }, + { + "label": "No", + "value": false + } + ], + "required": true + }, + "visible": true + } + ], + "steps": [ + { + "name": "clusterDetails", + "label": "Cluster Details", + "elements": [ + { + "name": "existingClusterSection", + "type": "Microsoft.Common.Section", + "elements": [ + { + "name": "clusterLookupControl", + "type": "Microsoft.Solutions.ArmApiControl", + "request": { + "method": "GET", + "path": "[concat(subscription().id, '/resourcegroups/', resourceGroup().name, '/providers/Microsoft.ContainerService/managedClusters?api-version=2022-03-01')]" + } + }, + { + "name": "existingClusterResourceName", + "type": "Microsoft.Common.DropDown", + "label": "AKS Cluster Name", + "toolTip": "The resource name of the existing AKS cluster.", + "constraints": { + "allowedValues": "[map(steps('clusterDetails').existingClusterSection.clusterLookupControl.value, (item) => parse(concat('{\"label\":\"', item.name, '\",\"value\":\"', item.name, '\"}')))]", + "required": true + } + } + ], + "visible": "[equals(basics('createNewCluster'), false)]" + }, + { + "name": "newClusterSection", + "type": "Microsoft.Common.Section", + "elements": [ + { + "name": "aksVersionLookupControl", + "type": "Microsoft.Solutions.ArmApiControl", + "request": { + "method": "GET", + "path": "[concat(subscription().id, '/providers/Microsoft.ContainerService/locations/', location(), '/orchestrators?api-version=2019-04-01&resource-type=managedClusters')]" + } + }, + { + "name": "newClusterResourceName", + "type": "Microsoft.Common.TextBox", + "label": "AKS cluster name", + "defaultValue": "", + "toolTip": "The resource name of the new AKS cluster. Use only allowed characters", + "constraints": { + "required": true, + "regex": "^[a-z0-9A-Z]{6,30}$", + "validationMessage": "Only alphanumeric characters are allowed, and the value must be 6-30 characters long." + } + }, + { + "name": "kubernetesVersion", + "type": "Microsoft.Common.DropDown", + "label": "Kubernetes version", + "toolTip": "The version of Kubernetes that should be used for this cluster. You will be able to upgrade this version after creating the cluster.", + "constraints": { + "allowedValues": "[map(steps('clusterDetails').newClusterSection.aksVersionLookupControl.properties.orchestrators, (item) => parse(concat('{\"label\":\"', item.orchestratorVersion, '\",\"value\":\"', item.orchestratorVersion, '\"}')))]", + "required": true + } + }, + { + "name": "vmSize", + "type": "Microsoft.Compute.SizeSelector", + "label": "VM size", + "toolTip": "The size of virtual machine of AKS worker nodes.", + "recommendedSizes": [ + "Standard_B4ms", + "Standard_DS2_v2", + "Standard_D4s_v3" + ], + "constraints": { + "allowedSizes": [ + "Standard_B4ms", + "Standard_DS2_v2", + "Standard_D4s_v3" + ], + "excludedSizes": [] + }, + "osPlatform": "Linux" + }, + { + "name": "osSKU", + "type": "Microsoft.Common.DropDown", + "label": "OS SKU", + "toolTip": "The SKU of Linux OS for VM.", + "defaultValue": "Ubuntu", + "constraints": { + "allowedValues": [ + { + "label": "Ubuntu", + "value": "Ubuntu" + }, + { + "label": "AzureLinux", + "value": "AzureLinux" + } + ], + "required": true + } + }, + { + "name": "enableAutoScaling", + "type": "Microsoft.Common.CheckBox", + "label": "Enable auto scaling", + "toolTip": "Enable auto scaling", + "defaultValue": true + }, + { + "name": "vmCount", + "type": "Microsoft.Common.Slider", + "min": 1, + "max": 10, + "label": "Number of AKS worker nodes", + "subLabel": "", + "defaultValue": 1, + "showStepMarkers": false, + "toolTip": "Specify the number of AKS worker nodes.", + "constraints": { + "required": false + }, + "visible": true + } + ], + "visible": "[basics('createNewCluster')]" + } + ] + }, + { + "name": "applicationDetails", + "label": "Application Details", + "elements": [ + { + "name": "extensionResourceName", + "type": "Microsoft.Common.TextBox", + "label": "Cluster extension resource name", + "defaultValue": "", + "toolTip": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long.", + "constraints": { + "required": true, + "regex": "^[a-z0-9]{6,30}$", + "validationMessage": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long." + }, + "visible": true + }, + { + "name": "extensionNamespace", + "type": "Microsoft.Common.TextBox", + "label": "Installation namespace", + "defaultValue": "nlk", + "toolTip": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long.", + "constraints": { + "required": true, + "regex": "^[a-z0-9]{3,30}$", + "validationMessage": "Only lowercase alphanumeric characters are allowed, and the value must be 6-30 characters long." + }, + "visible": true + }, + { + "name": "extensionAutoUpgrade", + "type": "Microsoft.Common.CheckBox", + "label": "Allow minor version upgrades of extension", + "toolTip": "Allow exntension to be upgraded automatically to latest minor version.", + "visible": true + }, + { + "name": "nginxaasDataplaneApiKey", + "type": "Microsoft.Common.TextBox", + "label": "NGINXaaS Dataplane API Key", + "defaultValue": "", + "toolTip": "The Dataplane API Key for your NGINXaaS for Azure deployment.", + "visible": true + }, + { + "name": "nginxaasDataplaneApiEndpoint", + "type": "Microsoft.Common.TextBox", + "label": "NGINXaaS Dataplane API Endpoint", + "defaultValue": "", + "toolTip": "The Dataplane API Endpoint for your NGINXaaS for Azure deployment.", + "visible": true + }, + { + "name": "additionalProductInfo", + "type": "Microsoft.Common.InfoBox", + "visible": true, + "options": { + "icon": "Info", + "text": "Learn more about NGINXaaS for Azure.", + "uri": "https://docs.nginx.com/nginxaas/azure/" + } + } + ] + } + ], + "outputs": { + "location": "[location()]", + "createNewCluster": "[basics('createNewCluster')]", + "clusterResourceName": "[if(basics('createNewCluster'), steps('clusterDetails').newClusterSection.newClusterResourceName, steps('clusterDetails').existingClusterSection.existingClusterResourceName)]", + "kubernetesVersion": "[steps('clusterDetails').newClusterSection.kubernetesVersion]", + "vmSize": "[steps('clusterDetails').newClusterSection.vmSize]", + "osSKU": "[steps('clusterDetails').newClusterSection.osSKU]", + "vmEnableAutoScale": "[steps('clusterDetails').newClusterSection.enableAutoScaling]", + "vmCount": "[steps('clusterDetails').newClusterSection.vmCount]", + "extensionResourceName": "[steps('applicationDetails').extensionResourceName]", + "extensionAutoUpgrade": "[steps('applicationDetails').extensionAutoUpgrade]", + "extensionNamespace": "[steps('applicationDetails').extensionNamespace]", + "nginxaasDataplaneApiKey": "[steps('applicationDetails').nginxaasDataplaneApiKey]", + "nginxaasDataplaneApiEndpoint": "[steps('applicationDetails').nginxaasDataplaneApiEndpoint]" + } + } +} From c052f01415f80c3f7c6cf9d9c0181e2bc0aae051 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 14:52:16 -0700 Subject: [PATCH 065/136] NLB-5733: Update Helm chart for Azure Marketplace Azure requires `global.azure.images` field in the Chart values so it can parse the Helm chart and copy them into an Azure-managed/owned ACR. This commit allows for non-Azure Marketplace users to continue to use the helm chart regularly and we will update the Chart values dynamically when we publish to markerplace to allow for newer workflows. --- charts/nlk/templates/_helpers.tpl | 8 +++++++- charts/nlk/templates/nlk-deployment.yaml | 3 +++ charts/nlk/values.yaml | 24 +++++++++++------------- version | 2 +- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/charts/nlk/templates/_helpers.tpl b/charts/nlk/templates/_helpers.tpl index 3f62fe9..6f5677d 100644 --- a/charts/nlk/templates/_helpers.tpl +++ b/charts/nlk/templates/_helpers.tpl @@ -91,14 +91,20 @@ Expand service account name. {{- end -}} {{- define "nlk.tag" -}} +{{- if .Values.global.azure -}} +{{- printf "%s" .Values.global.azure.images.nlk.tag -}} +{{- else -}} {{- default .Chart.AppVersion .Values.nlk.image.tag -}} {{- end -}} +{{- end -}} {{/* Expand image name. */}} {{- define "nlk.image" -}} -{{- if .Values.nlk.image.digest -}} +{{- if .Values.global.azure -}} +{{- printf "%s/%s:%s" .Values.global.azure.images.nlk.registry .Values.global.azure.images.nlk.repository (include "nlk.tag" .) -}} +{{- else if .Values.nlk.image.digest -}} {{- printf "%s/%s@%s" .Values.nlk.image.registry .Values.nlk.image.repository .Values.nlk.image.digest -}} {{- else -}} {{- printf "%s/%s:%s" .Values.nlk.image.registry .Values.nlk.image.repository (include "nlk.tag" .) -}} diff --git a/charts/nlk/templates/nlk-deployment.yaml b/charts/nlk/templates/nlk-deployment.yaml index 6270822..93fcd38 100644 --- a/charts/nlk/templates/nlk-deployment.yaml +++ b/charts/nlk/templates/nlk-deployment.yaml @@ -14,6 +14,9 @@ spec: metadata: labels: app: nlk +{{- if .Values.global.azure }} + azure-extensions-usage-release-identifier: {{ .Release.Name }} +{{- end }} annotations: checksum: {{ tpl (toYaml .Values.nlk) . | sha256sum }} spec: diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 75f6b54..4ce75ae 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -1,21 +1,27 @@ +##################################### +# Global Azure Marketplace configuration for AKS integration. +# DO NOT REMOVE +global: + azure: + # images: + # nlk: + # registry: registry-1.docker.io + # repository: nginx/nginxaas-loadbalancer-kubernetes + # tag: 0.4.0 +##################################### nlk: name: nginxaas-loadbalancer-kubernetes - kind: deployment - replicaCount: 1 - image: registry: registry-1.docker.io repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always ## Overrides the image tag whose default is the chart appVersion. tag: 0.4.0 - imagePullSecrets: [] nameOverride: "" fullnameOverride: "" - serviceAccount: ## Specifies whether a service account should be created create: true @@ -23,39 +29,31 @@ nlk: automount: true ## Annotations to add to the service account annotations: {} - config: ## trace,debug,info,warn,error,fatal,panic # logLevel: "warn" ## the nginx hosts (comma-separated) to send upstream updates to nginxHosts: "" - ## Sets the annotation value that NLK is looking for to watch a Service # serviceAnnotationMatch: nginxaas - tls: ## can also be set to "no-tls" to disable server cert verification mode: "ca-tls" - ## Override with your own NGINXaaS dataplane API Key. dataplaneApiKey: "test" - containerPort: http: 51031 - liveStatus: enable: true port: 51031 initialDelaySeconds: 5 periodSeconds: 2 - readyStatus: enable: true port: 51031 initialDelaySeconds: 5 periodSeconds: 2 - rbac: ## Configures RBAC. create: true diff --git a/version b/version index 267577d..8f0916f 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.4.1 +0.5.0 From 1195350318abdf72725428b1b23164309cbed0e2 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 14:52:16 -0700 Subject: [PATCH 066/136] NLB-5733: Update Helm chart for Azure Marketplace Azure requires `global.azure.images` field in the Chart values so it can parse the Helm chart and copy them into an Azure-managed/owned ACR. This commit allows for non-Azure Marketplace users to continue to use the helm chart regularly and we will update the Chart values dynamically when we publish to markerplace to allow for newer workflows. --- charts/nlk/templates/_helpers.tpl | 2 +- charts/nlk/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/nlk/templates/_helpers.tpl b/charts/nlk/templates/_helpers.tpl index 6f5677d..27dbe94 100644 --- a/charts/nlk/templates/_helpers.tpl +++ b/charts/nlk/templates/_helpers.tpl @@ -103,7 +103,7 @@ Expand image name. */}} {{- define "nlk.image" -}} {{- if .Values.global.azure -}} -{{- printf "%s/%s:%s" .Values.global.azure.images.nlk.registry .Values.global.azure.images.nlk.repository (include "nlk.tag" .) -}} +{{- printf "%s/%s:%s" .Values.global.azure.images.nlk.registry .Values.global.azure.images.nlk.image (include "nlk.tag" .) -}} {{- else if .Values.nlk.image.digest -}} {{- printf "%s/%s@%s" .Values.nlk.image.registry .Values.nlk.image.repository .Values.nlk.image.digest -}} {{- else -}} diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 4ce75ae..f5be244 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -6,7 +6,7 @@ global: # images: # nlk: # registry: registry-1.docker.io - # repository: nginx/nginxaas-loadbalancer-kubernetes + # image: nginx/nginxaas-loadbalancer-kubernetes # tag: 0.4.0 ##################################### nlk: From 60352fa3924372de35cb56afb58710a50fe92265 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 17:47:50 -0700 Subject: [PATCH 067/136] NLB-5708: Create ARM template for AKS extension This defines the resources that will get created as part of the extension deployment. For the time being, Azure only allows creating AKS and extensions. --- charts/armTemplate.json | 256 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 charts/armTemplate.json diff --git a/charts/armTemplate.json b/charts/armTemplate.json new file mode 100644 index 0000000..1353fdb --- /dev/null +++ b/charts/armTemplate.json @@ -0,0 +1,256 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "extensionResourceName": { + "type": "string", + "metadata": { + "description": "The name of the extension." + } + }, + "extensionNamespace": { + "type": "string", + "defaultValue": "nlk" + }, + "clusterResourceName": { + "type": "String", + "metadata": { + "description": "The name of the Managed Cluster resource." + } + }, + "createNewCluster": { + "type": "Bool", + "defaultValue": true, + "metadata": { + "description": "When set to 'true', creates new AKS cluster. Otherwise, an existing cluster is used." + } + }, + "location": { + "type": "String", + "metadata": { + "description": "The location of AKS resource." + } + }, + "extensionAutoUpgrade": { + "defaultValue": false, + "metadata": { + "description": "Allow auto upgrade of minor version for the extension." + }, + "type": "bool" + }, + "nginxaasDataplaneApiKey": { + "type": "String" + }, + "nginxaasDataplaneApiEndpoint": { + "type": "String" + }, + "vmSize": { + "type": "String", + "defaultValue": "Standard_DS2_v2", + "metadata": { + "description": "VM size" + } + }, + "vmEnableAutoScale": { + "type": "Bool", + "defaultValue": true, + "metadata": { + "description": "enable auto scaling" + } + }, + "vmCount": { + "type": "Int", + "defaultValue": 3, + "metadata": { + "description": "VM count" + } + }, + "dnsPrefix": { + "defaultValue": "[concat(parameters('clusterResourceName'),'-dns')]", + "type": "String", + "metadata": { + "description": "Optional DNS prefix to use with hosted Kubernetes API server FQDN." + } + }, + "osDiskSizeGB": { + "defaultValue": 0, + "minValue": 0, + "maxValue": 1023, + "type": "Int", + "metadata": { + "description": "Disk size (in GiB) to provision for each of the agent pool nodes. This value ranges from 0 to 1023. Specifying 0 will apply the default disk size for that agentVMSize." + } + }, + "kubernetesVersion": { + "type": "String", + "defaultValue": "1.26.3", + "metadata": { + "description": "The version of Kubernetes." + } + }, + "networkPlugin": { + "defaultValue": "kubenet", + "allowedValues": [ + "azure", + "kubenet" + ], + "type": "String", + "metadata": { + "description": "Network plugin used for building Kubernetes network." + } + }, + "enableRBAC": { + "defaultValue": true, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off of RBAC." + } + }, + "enablePrivateCluster": { + "defaultValue": false, + "type": "Bool", + "metadata": { + "description": "Enable private network access to the Kubernetes cluster." + } + }, + "enableHttpApplicationRouting": { + "defaultValue": true, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off http application routing." + } + }, + "enableAzurePolicy": { + "defaultValue": false, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off Azure Policy addon." + } + }, + "enableSecretStoreCSIDriver": { + "defaultValue": false, + "type": "Bool", + "metadata": { + "description": "Boolean flag to turn on and off secret store CSI driver." + } + }, + "osSKU": { + "type": "string", + "defaultValue": "AzureLinux", + "allowedValues": [ + "AzureLinux", + "Ubuntu" + ], + "metadata": { + "description": "The Linux SKU to use." + } + }, + "enableFIPS": { + "type": "Bool", + "defaultValue": true, + "metadata": { + "description": "Enable FIPS. https://learn.microsoft.com/en-us/azure/aks/create-node-pools#fips-enabled-node-pools" + } + } + }, + "variables": { + "plan-name": "DONOTMODIFY", + "plan-publisher": "DONOTMODIFY", + "plan-offerID": "DONOTMODIFY", + "releaseTrain": "DONOTMODIFY", + "clusterExtensionTypeName": "DONOTMODIFY" + }, + "resources": [ + { + "type": "Microsoft.ContainerService/managedClusters", + "condition": "[parameters('createNewCluster')]", + "apiVersion": "2022-11-01", + "name": "[parameters('clusterResourceName')]", + "location": "[parameters('location')]", + "dependsOn": [], + "tags": {}, + "sku": { + "name": "Basic", + "tier": "Free" + }, + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "kubernetesVersion": "[parameters('kubernetesVersion')]", + "enableRBAC": "[parameters('enableRBAC')]", + "dnsPrefix": "[parameters('dnsPrefix')]", + "agentPoolProfiles": [ + { + "name": "agentpool", + "osDiskSizeGB": "[parameters('osDiskSizeGB')]", + "count": "[parameters('vmCount')]", + "enableAutoScaling": "[parameters('vmEnableAutoScale')]", + "enableFIPS": "[parameters('enableFIPS')]", + "minCount": "[if(parameters('vmEnableAutoScale'), 1, json('null'))]", + "maxCount": "[if(parameters('vmEnableAutoScale'), 10, json('null'))]", + "vmSize": "[parameters('vmSize')]", + "osType": "Linux", + "osSKU": "[parameters('osSKU')]", + "storageProfile": "ManagedDisks", + "type": "VirtualMachineScaleSets", + "mode": "System", + "maxPods": 110, + "availabilityZones": [], + "enableNodePublicIP": false, + "tags": {} + } + ], + "networkProfile": { + "loadBalancerSku": "standard", + "networkPlugin": "[parameters('networkPlugin')]" + }, + "apiServerAccessProfile": { + "enablePrivateCluster": "[parameters('enablePrivateCluster')]" + }, + "addonProfiles": { + "httpApplicationRouting": { + "enabled": "[parameters('enableHttpApplicationRouting')]" + }, + "azurepolicy": { + "enabled": "[parameters('enableAzurePolicy')]" + }, + "azureKeyvaultSecretsProvider": { + "enabled": "[parameters('enableSecretStoreCSIDriver')]" + } + } + } + }, + { + "type": "Microsoft.KubernetesConfiguration/extensions", + "apiVersion": "2023-05-01", + "name": "[parameters('extensionResourceName')]", + "properties": { + "extensionType": "[variables('clusterExtensionTypeName')]", + "autoUpgradeMinorVersion": "[parameters('extensionAutoUpgrade')]", + "releaseTrain": "[variables('releaseTrain')]", + "configurationSettings": { + "nlk.config.dataplaneApiKey": "[parameters('nginxaasDataplaneApiKey')]", + "nlk.config.nginxHosts": "[parameters('nginxaasDataplaneApiEndpoint')]" + }, + "configurationProtectedSettings": {}, + "scope": { + "namespace": { + "targetNamespace": "[parameters('extensionNamespace')]" + } + } + }, + "plan": { + "name": "[variables('plan-name')]", + "publisher": "[variables('plan-publisher')]", + "product": "[variables('plan-offerID')]" + }, + "scope": "[concat('Microsoft.ContainerService/managedClusters/', parameters('clusterResourceName'))]", + "dependsOn": [ + "[resourceId('Microsoft.ContainerService/managedClusters/', parameters('clusterResourceName'))]" + ] + } + ], + "outputs": { + } +} From db737278b51ca46389370c5fcd8aa492f4f422a9 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 10 Oct 2024 19:22:06 -0700 Subject: [PATCH 068/136] Add CNAB validation We should make sure to validate the CNAB bundle as part of regular work, especially while we are iterating on NLK. --- .gitlab-ci.yml | 15 ++++++++++++ Makefile | 8 +++++++ scripts/cnab.sh | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100755 scripts/cnab.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d471ae7..71268a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,7 @@ stages: variables: DEVTOOLS_IMG: ${DEVOPS_DOCKER_URL_DEFAULT}/nginx-azure-lb/nlb-devtools:latest + CNAB_IMG: mcr.microsoft.com/container-package-app:latest workflow: rules: @@ -110,6 +111,20 @@ lint + unit-test + build: when: never - if: '$CI_COMMIT_BRANCH || $CI_MERGE_REQUEST_ID' +validate-cnab: + stage: lint+test+build + image: $CNAB_IMG + extends: + - .devops-docker-cicd-large + script: + - tdnf install -y make wget + - wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq && chmod +x /usr/local/bin/yq + - make cnab-validate + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: '$CI_COMMIT_BRANCH || $CI_MERGE_REQUEST_ID' + unit-test-data-race: variables: GO_DATA_RACE: "true" diff --git a/Makefile b/Makefile index e53c440..08e07ce 100644 --- a/Makefile +++ b/Makefile @@ -60,3 +60,11 @@ publish: build-linux build-linux-docker clean: rm -rf $(BUILD_DIR)/ + +.PHONY: cnab-validate cnab-package + +cnab-validate: + @./scripts/cnab.sh validate + +cnab-package: + @./scripts/cnab.sh package diff --git a/scripts/cnab.sh b/scripts/cnab.sh new file mode 100755 index 0000000..f8cc777 --- /dev/null +++ b/scripts/cnab.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +set -euo pipefail + +log() { + printf "\033[0;36m${*}\033[0m\n" >&2 +} + +package() { + log "Not Implemented." + exit 1 +} + +validate() { + CMD="cpa verify -d ${BUNDLE_DIR} --telemetryOptOut" + ${CMD} + +} + +set_version() { + VERSION=$(cat version) +} + +update_helm_chart() { + yq -ie '.global.azure.images.nlk.registry = .nlk.image.registry | .global.azure.images.nlk.image = .nlk.image.repository | .global.azure.images.nlk.tag = env(VERSION)' charts/nlk/values.yaml +} + +update_bundle() { + yq -ie '.version = env(VERSION)' charts/manifest.yaml +} + +check_ci() { + if [[ "$CI" != "true" ]]; then + log "This script should be the run in the CI only." + exit 1 + fi +} + +set_vars() { + BUNDLE_DIR="${CI_PROJECT_DIR}/charts/" +} + +main() { + check_ci + set_vars + set_version + update_helm_chart + update_bundle + local action="$1" + case "$action" in + validate) + validate + ;; + package) + package + ;; + *) + log "Action not supported." + exit 1 + ;; + esac +} + +main "$@" From 514fafaaa3a1a36c2813dac52d761f33827dab83 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 15 Oct 2024 12:59:55 -0700 Subject: [PATCH 069/136] Add constraints to dataplane-related user inputs contraints are required on textboxes. --- charts/createUIDefinition.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/charts/createUIDefinition.json b/charts/createUIDefinition.json index 906e08e..c7566d2 100644 --- a/charts/createUIDefinition.json +++ b/charts/createUIDefinition.json @@ -212,6 +212,11 @@ "label": "NGINXaaS Dataplane API Key", "defaultValue": "", "toolTip": "The Dataplane API Key for your NGINXaaS for Azure deployment.", + "constraints": { + "required": false, + "regex": ".*", + "validationMessage": "Use the dataplane API key for your deployment." + }, "visible": true }, { @@ -220,6 +225,11 @@ "label": "NGINXaaS Dataplane API Endpoint", "defaultValue": "", "toolTip": "The Dataplane API Endpoint for your NGINXaaS for Azure deployment.", + "constraints": { + "required": false, + "regex": ".*", + "validationMessage": "Retreive the dataplane API endpoint from your deployment." + }, "visible": true }, { From 0b18aa3f267cd91717b9f766d95e36d39f4473ca Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 15 Oct 2024 18:58:56 -0700 Subject: [PATCH 070/136] Allow releasing CNAB We need to release the CNAB package to our own ACR for the Azure-owned marketplace ACR to pick it up. This commit allows for us to publish the CNAB package as part of the release process this enabling continuous deployment to Azure Marketplace. --- .gitlab-ci.yml | 16 ++++++++++++++-- scripts/cnab.sh | 5 +++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71268a2..8237868 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,7 @@ stages: - lint+test+build - security scanning - release + - release-cnab variables: DEVTOOLS_IMG: ${DEVOPS_DOCKER_URL_DEFAULT}/nginx-azure-lb/nlb-devtools:latest @@ -117,8 +118,8 @@ validate-cnab: extends: - .devops-docker-cicd-large script: - - tdnf install -y make wget - - wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq && chmod +x /usr/local/bin/yq + - tdnf install -y make + - curl -L https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq - make cnab-validate rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' @@ -244,3 +245,14 @@ dockerhub-helm-release: - .release-common script: - ./scripts/release.sh helm-chart + +cnab-release: + extends: + - .devops-docker-cicd + - .release-common + stage: release-cnab + image: $CNAB_IMG + script: + - tdnf install -y make wget + - curl -L https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + - make cnab-package diff --git a/scripts/cnab.sh b/scripts/cnab.sh index f8cc777..bac8a33 100755 --- a/scripts/cnab.sh +++ b/scripts/cnab.sh @@ -7,8 +7,9 @@ log() { } package() { - log "Not Implemented." - exit 1 + CMD="az acr login --name nlbmarketplaceacrprod --username ${ARM_CLIENT_ID_ACR} --password ${ARM_CLIENT_SECRET_ACR}" + CMD+=" && cpa buildbundle -d ${BUNDLE_DIR} --telemetryOptOut" + ${CMD} } validate() { From 6b787d3f022a37b23c0be158af95f94c4759efad Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 16 Oct 2024 14:19:28 -0700 Subject: [PATCH 071/136] Fix cnab publish commands Splitting up the publish commnds as bash is taking the cpa buildbundle command as an arg to az acr. --- scripts/cnab.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/cnab.sh b/scripts/cnab.sh index bac8a33..7d3c0b0 100755 --- a/scripts/cnab.sh +++ b/scripts/cnab.sh @@ -8,7 +8,8 @@ log() { package() { CMD="az acr login --name nlbmarketplaceacrprod --username ${ARM_CLIENT_ID_ACR} --password ${ARM_CLIENT_SECRET_ACR}" - CMD+=" && cpa buildbundle -d ${BUNDLE_DIR} --telemetryOptOut" + ${CMD} + CMD="cpa buildbundle -d ${BUNDLE_DIR} --telemetryOptOut" ${CMD} } From cfb92d980e55d0457ba004e9fcf9bfa9b1e12a04 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 17 Oct 2024 12:13:49 -0700 Subject: [PATCH 072/136] Change extension to ClusterScope When trying to install the extension in a namespaced mode (which is the default on the current offer/plan), the AKS extension machinery creates a k8s service account (different from the NLK service account) which is restricted to the extension namespace. Due to the aforementioned, the service account cannot read Cluster Roles that we create for NLK causing the extension to fail. Switching over to the cluster level extension will help here as when the AKS extension gets installed cluster-wide, the undelying service account has all the necessary permissions for the extension to be installed. --- charts/manifest.yaml | 3 ++- version | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/manifest.yaml b/charts/manifest.yaml index 9138b81..6883ea9 100644 --- a/charts/manifest.yaml +++ b/charts/manifest.yaml @@ -7,4 +7,5 @@ clusterArmTemplate: "./armTemplate.json" uiDefinition: "./createUIDefinition.json" registryServer: "nlbmarketplaceacrprod.azurecr.io" extensionRegistrationParameters: - defaultScope: "namespace" + defaultScope: "cluster" + namespace: "nlk" diff --git a/version b/version index 8f0916f..4b9fcbe 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.5.0 +0.5.1 From 2da6ed5132caac1a55f796f8ce21f5615b627608 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 21 Oct 2024 10:14:45 -0700 Subject: [PATCH 073/136] Use release namespace everywhere This was missed as part of the manifest rollout. --- charts/armTemplate.json | 4 ++-- version | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/armTemplate.json b/charts/armTemplate.json index 1353fdb..ae3b699 100644 --- a/charts/armTemplate.json +++ b/charts/armTemplate.json @@ -235,8 +235,8 @@ }, "configurationProtectedSettings": {}, "scope": { - "namespace": { - "targetNamespace": "[parameters('extensionNamespace')]" + "cluster": { + "releaseNamespace": "[parameters('extensionNamespace')]" } } }, diff --git a/version b/version index 4b9fcbe..cb0c939 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.5.1 +0.5.2 From 15b19e450b35edb3655064fec0d24d037934e90f Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 21 Oct 2024 16:31:39 -0700 Subject: [PATCH 074/136] Fixup helm value supplied in extension It is `nlk.dataplaneApiKey` and not `nlk.config.dataplaneApiKey`. --- charts/armTemplate.json | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/armTemplate.json b/charts/armTemplate.json index ae3b699..510c784 100644 --- a/charts/armTemplate.json +++ b/charts/armTemplate.json @@ -230,7 +230,7 @@ "autoUpgradeMinorVersion": "[parameters('extensionAutoUpgrade')]", "releaseTrain": "[variables('releaseTrain')]", "configurationSettings": { - "nlk.config.dataplaneApiKey": "[parameters('nginxaasDataplaneApiKey')]", + "nlk.dataplaneApiKey": "[parameters('nginxaasDataplaneApiKey')]", "nlk.config.nginxHosts": "[parameters('nginxaasDataplaneApiEndpoint')]" }, "configurationProtectedSettings": {}, diff --git a/version b/version index cb0c939..be14282 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.5.2 +0.5.3 From 21c42c7b7431bad04f28b92cdd9c679528904ba3 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 18 Oct 2024 15:52:36 -0700 Subject: [PATCH 075/136] Allow NLK to READ endpoint slices We will need to read endpoint slices in AKS to support the use-case for Cluster IP using Azure CNI. --- charts/nlk/templates/clusterrole.yaml | 8 ++++++++ version | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/charts/nlk/templates/clusterrole.yaml b/charts/nlk/templates/clusterrole.yaml index c652c4c..47f95dd 100644 --- a/charts/nlk/templates/clusterrole.yaml +++ b/charts/nlk/templates/clusterrole.yaml @@ -13,4 +13,12 @@ rules: - get - list - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch {{- end }} diff --git a/version b/version index be14282..a918a2a 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.5.3 +0.6.0 From b4ede3296be4cc179cb7fd832d9159d63662c64c Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 16 Oct 2024 15:15:03 -0600 Subject: [PATCH 076/136] NLB-5753 Added buildinfo package and injected vars at build time The code will then be able to reference build information, such as semver, easily at run time. --- pkg/buildinfo/buildinfo.go | 15 +++++++++++++++ scripts/build.sh | 4 +--- 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 pkg/buildinfo/buildinfo.go diff --git a/pkg/buildinfo/buildinfo.go b/pkg/buildinfo/buildinfo.go new file mode 100644 index 0000000..5d8839d --- /dev/null +++ b/pkg/buildinfo/buildinfo.go @@ -0,0 +1,15 @@ +package buildinfo + +var semVer string + +// SemVer is the version number of this build as provided by build pipeline +func SemVer() string { + return semVer +} + +var shortHash string + +// ShortHash is the 8 char git shorthash +func ShortHash() string { + return shortHash +} diff --git a/scripts/build.sh b/scripts/build.sh index fb1a634..c071a19 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,7 +4,6 @@ set -ex os="$1" -GIT_DESCRIBE=$(git describe --long --always --dirty) if [[ -z $CI_COMMIT_SHORT_SHA ]]; then CI_COMMIT_SHORT_SHA=$(git rev-parse --short=8 HEAD) fi @@ -21,13 +20,12 @@ fi mkdir -p "$BUILD_DIR" pkg_path="./cmd/nginx-loadbalancer-kubernetes" -BUILDPKG="gitlab.com/f5/nginx/nginxazurelb/nlk/pkg/buildinfo" +BUILDPKG="github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" ldflags=( # Set the value of the string variable in importpath named name to value. -X "'$BUILDPKG.semVer=$VERSION'" -X "'$BUILDPKG.shortHash=$CI_COMMIT_SHORT_SHA'" - -X "'$BUILDPKG.gitDescribe=$GIT_DESCRIBE'" -s # Omit the symbol table and debug information. -w # Omit the DWARF symbol table. -extldflags "'-fno-PIC'" From e2f0b06c2c96bfd15013a8c9b1be94b0c582a0c7 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 16 Oct 2024 15:16:14 -0600 Subject: [PATCH 077/136] NLB-5753 Replaced logrus with slog as the nlk logger; added version info to all log lines Go's slog now provides comparable functionality to logrus while being more up-to-date. Added version to all log lines. --- cmd/certificates-test-harness/main.go | 14 ++++--- cmd/nginx-loadbalancer-kubernetes/main.go | 46 ++++++++++----------- cmd/tls-config-factory-test-harness/main.go | 20 +++++---- go.mod | 1 - go.sum | 4 -- internal/application/border_client.go | 7 ++-- internal/application/null_border_client.go | 10 ++--- internal/authentication/factory.go | 18 ++++---- internal/certification/certificates.go | 28 ++++++------- internal/communication/factory.go | 4 +- internal/configuration/settings.go | 4 +- internal/observation/handler.go | 19 ++++----- internal/observation/watcher.go | 30 +++++++------- internal/probation/server.go | 16 ++++--- internal/probation/server_test.go | 4 +- internal/synchronization/synchronizer.go | 40 +++++++++--------- internal/translation/translator.go | 10 ++--- 17 files changed, 135 insertions(+), 140 deletions(-) diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go index 056d303..f3468a9 100644 --- a/cmd/certificates-test-harness/main.go +++ b/cmd/certificates-test-harness/main.go @@ -4,10 +4,11 @@ import ( "context" "errors" "fmt" + "log/slog" + "os" "path/filepath" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -15,15 +16,18 @@ import ( ) func main() { - logrus.SetLevel(logrus.DebugLevel) + handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + logger := slog.New(handler) + slog.SetDefault(logger) err := run() if err != nil { - logrus.Fatal(err) + slog.Error(err.Error()) + os.Exit(1) } } func run() error { - logrus.Info("certificates-test-harness::run") + slog.Info("certificates-test-harness::run") ctx := context.Background() var err error @@ -47,7 +51,7 @@ func run() error { } func buildKubernetesClient() (*kubernetes.Clientset, error) { - logrus.Debug("Watcher::buildKubernetesClient") + slog.Debug("Watcher::buildKubernetesClient") var kubeconfig *string var k8sConfig *rest.Config diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index a9f6cd9..8064348 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -8,12 +8,14 @@ package main import ( "context" "fmt" + "log/slog" + "os" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/observation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/probation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" - "github.com/sirupsen/logrus" + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" @@ -22,7 +24,8 @@ import ( func main() { err := run() if err != nil { - logrus.Fatal(err) + slog.Error(err.Error()) + os.Exit(1) } } @@ -40,7 +43,7 @@ func run() error { return fmt.Errorf(`error occurred accessing configuration: %w`, err) } - setLogLevel(settings.LogLevel) + initializeLogger(settings.LogLevel) synchronizerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) @@ -78,37 +81,30 @@ func run() error { return nil } -func setLogLevel(logLevel string) { - logrus.Debugf("Settings::setLogLevel: %s", logLevel) - switch logLevel { - case "panic": - logrus.SetLevel(logrus.PanicLevel) - - case "fatal": - logrus.SetLevel(logrus.FatalLevel) +func initializeLogger(logLevel string) { + programLevel := new(slog.LevelVar) + switch logLevel { case "error": - logrus.SetLevel(logrus.ErrorLevel) - + programLevel.Set(slog.LevelError) case "warn": - logrus.SetLevel(logrus.WarnLevel) - + programLevel.Set(slog.LevelWarn) case "info": - logrus.SetLevel(logrus.InfoLevel) - + programLevel.Set(slog.LevelInfo) case "debug": - logrus.SetLevel(logrus.DebugLevel) - - case "trace": - logrus.SetLevel(logrus.TraceLevel) - + programLevel.Set(slog.LevelDebug) default: - logrus.SetLevel(logrus.WarnLevel) + programLevel.Set(slog.LevelWarn) } + + handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: programLevel}) + logger := slog.New(handler).With("version", buildinfo.SemVer()) + slog.SetDefault(logger) + slog.Debug("Settings::setLogLevel", slog.String("level", logLevel)) } func buildKubernetesClient() (*kubernetes.Clientset, error) { - logrus.Debug("Watcher::buildKubernetesClient") + slog.Debug("Watcher::buildKubernetesClient") k8sConfig, err := rest.InClusterConfig() if err == rest.ErrNotInCluster { return nil, fmt.Errorf(`not running in a Cluster: %w`, err) @@ -125,7 +121,7 @@ func buildKubernetesClient() (*kubernetes.Clientset, error) { } func buildWorkQueue(settings configuration.WorkQueueSettings) workqueue.RateLimitingInterface { - logrus.Debug("Watcher::buildSynchronizerWorkQueue") + slog.Debug("Watcher::buildSynchronizerWorkQueue") rateLimiter := workqueue.NewItemExponentialFailureRateLimiter(settings.RateLimiterBase, settings.RateLimiterMax) return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name) diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 7b853e0..7883d65 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -2,13 +2,13 @@ package main import ( "bufio" + "log/slog" "os" "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" ) const ( @@ -22,14 +22,16 @@ type TLSConfiguration struct { } func main() { - logrus.SetLevel(logrus.DebugLevel) + handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) + logger := slog.New(handler) + slog.SetDefault(logger) configurations := buildConfigMap() for name, settings := range configurations { - logrus.Infof("\033[H\033[2J") + slog.Info("\033[H\033[2J") - logrus.Infof("\n\n\t*** Building TLS config for <<< %s >>>\n\n", name) + slog.Info("\n\n\t*** Building TLS config\n\n", "name", name) tlsConfig, err := authentication.NewTLSConfig(settings.Settings) if err != nil { @@ -47,15 +49,17 @@ func main() { certificateCount = len(tlsConfig.Certificates) } - logrus.Infof("Successfully built TLS config: \n\tDescription: %s \n\tRootCA count: %v\n\tCertificate count: %v", - settings.Description, rootCaCount, certificateCount, + slog.Info("Successfully built TLS config", + "description", settings.Description, + "rootCaCount", rootCaCount, + "certificateCount", certificateCount, ) _, _ = bufio.NewReader(os.Stdin).ReadBytes('\n') } - logrus.Infof("\033[H\033[2J") - logrus.Infof("\n\n\t*** All done! ***\n\n") + slog.Info("\033[H\033[2J") + slog.Info("\n\n\t*** All done! ***\n\n") } func buildConfigMap() map[string]TLSConfiguration { diff --git a/go.mod b/go.mod index 056c7e2..94a1319 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ toolchain go1.21.4 require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 - github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 k8s.io/api v0.26.0 diff --git a/go.sum b/go.sum index 098adea..1433805 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,6 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -134,7 +132,6 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -190,7 +187,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/internal/application/border_client.go b/internal/application/border_client.go index a510953..e4eded8 100644 --- a/internal/application/border_client.go +++ b/internal/application/border_client.go @@ -7,9 +7,9 @@ package application import ( "fmt" + "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" ) // Interface defines the functions required to implement a Border Client. @@ -19,8 +19,7 @@ type Interface interface { } // BorderClient defines any state need by the Border Client. -type BorderClient struct { -} +type BorderClient struct{} // NewBorderClient is the Factory function for creating a Border Client. // @@ -29,7 +28,7 @@ type BorderClient struct { // 2. Add a new constant in application_constants.go that acts as a key for selecting the client; // 3. Update the NewBorderClient factory method in border_client.go that returns the client; func NewBorderClient(clientType string, borderClient interface{}) (Interface, error) { - logrus.Debugf(`NewBorderClient for type: %s`, clientType) + slog.Debug("NewBorderClient", slog.String("client", clientType)) switch clientType { case ClientTypeNginxStream: diff --git a/internal/application/null_border_client.go b/internal/application/null_border_client.go index b59ca22..295ca62 100644 --- a/internal/application/null_border_client.go +++ b/internal/application/null_border_client.go @@ -6,15 +6,15 @@ package application import ( + "log/slog" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" ) // NullBorderClient is a BorderClient that does nothing. // It serves only to prevent a panic if the BorderClient // is not set correctly and errors from the factory methods are ignored. -type NullBorderClient struct { -} +type NullBorderClient struct{} // NewNullBorderClient is the Factory function for creating a NullBorderClient func NewNullBorderClient() (Interface, error) { @@ -23,12 +23,12 @@ func NewNullBorderClient() (Interface, error) { // Update logs a Warning. It is, after all, a NullObject Pattern implementation. func (nbc *NullBorderClient) Update(_ *core.ServerUpdateEvent) error { - logrus.Warn("NullBorderClient.Update called") + slog.Warn("NullBorderClient.Update called") return nil } // Delete logs a Warning. It is, after all, a NullObject Pattern implementation. func (nbc *NullBorderClient) Delete(_ *core.ServerUpdateEvent) error { - logrus.Warn("NullBorderClient.Delete called") + slog.Warn("NullBorderClient.Delete called") return nil } diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go index a51d7ab..c0664f6 100644 --- a/internal/authentication/factory.go +++ b/internal/authentication/factory.go @@ -12,14 +12,14 @@ import ( "crypto/x509" "encoding/pem" "fmt" + "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/sirupsen/logrus" ) func NewTLSConfig(settings configuration.Settings) (*tls.Config, error) { - logrus.Debugf("authentication::NewTLSConfig Creating TLS config for mode: '%s'", settings.TLSMode) + slog.Debug("authentication::NewTLSConfig Creating TLS config", "mode", settings.TLSMode) switch settings.TLSMode { case configuration.NoTLS: @@ -43,7 +43,7 @@ func NewTLSConfig(settings configuration.Settings) (*tls.Config, error) { } func buildSelfSignedTLSConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildSelfSignedTlsConfig Building self-signed TLS config") + slog.Debug("authentication::buildSelfSignedTlsConfig Building self-signed TLS config") certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) if err != nil { return nil, err @@ -57,7 +57,7 @@ func buildSelfSignedTLSConfig(certificates *certification.Certificates) (*tls.Co } func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildSelfSignedMtlsConfig Building self-signed mTLS config") + slog.Debug("authentication::buildSelfSignedMtlsConfig Building self-signed mTLS config") certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) if err != nil { return nil, err @@ -67,7 +67,7 @@ func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.C if err != nil { return nil, err } - logrus.Debugf("buildSelfSignedMtlsConfig Certificate: %v", certificate) + slog.Debug("buildSelfSignedMtlsConfig Certificate", "certificate", certificate) //nolint:gosec return &tls.Config{ @@ -79,14 +79,14 @@ func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.C } func buildBasicTLSConfig(skipVerify bool) *tls.Config { - logrus.Debugf("authentication::buildBasicTLSConfig skipVerify(%v)", skipVerify) + slog.Debug("authentication::buildBasicTLSConfig", slog.Bool("skipVerify", skipVerify)) return &tls.Config{ InsecureSkipVerify: skipVerify, //nolint:gosec } } func buildCATLSConfig(certificates *certification.Certificates) (*tls.Config, error) { - logrus.Debug("authentication::buildCATLSConfig") + slog.Debug("authentication::buildCATLSConfig") certificate, err := buildCertificates(certificates.GetClientCertificate()) if err != nil { return nil, err @@ -100,12 +100,12 @@ func buildCATLSConfig(certificates *certification.Certificates) (*tls.Config, er } func buildCertificates(privateKeyPEM []byte, certificatePEM []byte) (tls.Certificate, error) { - logrus.Debug("authentication::buildCertificates") + slog.Debug("authentication::buildCertificates") return tls.X509KeyPair(certificatePEM, privateKeyPEM) } func buildCaCertificatePool(caCert []byte) (*x509.CertPool, error) { - logrus.Debug("authentication::buildCaCertificatePool") + slog.Debug("authentication::buildCaCertificatePool") block, _ := pem.Decode(caCert) if block == nil { return nil, fmt.Errorf("failed to decode PEM block containing CA certificate") diff --git a/internal/certification/certificates.go b/internal/certification/certificates.go index 3ecbc46..c59eb0e 100644 --- a/internal/certification/certificates.go +++ b/internal/certification/certificates.go @@ -12,8 +12,8 @@ package certification import ( "context" "fmt" + "log/slog" - "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -81,7 +81,7 @@ func (c *Certificates) GetClientCertificate() (core.SecretBytes, core.SecretByte // Initialize initializes the Certificates object. Sets up a SharedInformer for the Secrets Resource. func (c *Certificates) Initialize() error { - logrus.Info("Certificates::Initialize") + slog.Info("Certificates::Initialize") var err error @@ -101,7 +101,7 @@ func (c *Certificates) Initialize() error { // Run starts the SharedInformer. func (c *Certificates) Run() error { - logrus.Info("Certificates::Run") + slog.Info("Certificates::Run") if c.informer == nil { return fmt.Errorf(`initialize must be called before Run`) @@ -115,7 +115,7 @@ func (c *Certificates) Run() error { } func (c *Certificates) buildInformer() cache.SharedInformer { - logrus.Debug("Certificates::buildInformer") + slog.Debug("Certificates::buildInformer") options := informers.WithNamespace(SecretsNamespace) factory := informers.NewSharedInformerFactoryWithOptions(c.k8sClient, 0, options) @@ -125,7 +125,7 @@ func (c *Certificates) buildInformer() cache.SharedInformer { } func (c *Certificates) initializeEventHandlers() error { - logrus.Debug("Certificates::initializeEventHandlers") + slog.Debug("Certificates::initializeEventHandlers") var err error @@ -144,11 +144,11 @@ func (c *Certificates) initializeEventHandlers() error { } func (c *Certificates) handleAddEvent(obj interface{}) { - logrus.Debug("Certificates::handleAddEvent") + slog.Debug("Certificates::handleAddEvent") secret, ok := obj.(*corev1.Secret) if !ok { - logrus.Errorf("Certificates::handleAddEvent: unable to cast object to Secret") + slog.Error("Certificates::handleAddEvent: unable to cast object to Secret") return } @@ -162,15 +162,15 @@ func (c *Certificates) handleAddEvent(obj interface{}) { c.Certificates[secret.Name][k] = core.SecretBytes(v) } - logrus.Debugf("Certificates::handleAddEvent: certificates (%d)", len(c.Certificates)) + slog.Debug("Certificates::handleAddEvent", slog.Int("certCount", len(c.Certificates))) } func (c *Certificates) handleDeleteEvent(obj interface{}) { - logrus.Debug("Certificates::handleDeleteEvent") + slog.Debug("Certificates::handleDeleteEvent") secret, ok := obj.(*corev1.Secret) if !ok { - logrus.Errorf("Certificates::handleDeleteEvent: unable to cast object to Secret") + slog.Error("Certificates::handleDeleteEvent: unable to cast object to Secret") return } @@ -178,15 +178,15 @@ func (c *Certificates) handleDeleteEvent(obj interface{}) { delete(c.Certificates, secret.Name) } - logrus.Debugf("Certificates::handleDeleteEvent: certificates (%d)", len(c.Certificates)) + slog.Debug("Certificates::handleDeleteEvent", slog.Int("certCount", len(c.Certificates))) } func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { - logrus.Debug("Certificates::handleUpdateEvent") + slog.Debug("Certificates::handleUpdateEvent") secret, ok := newValue.(*corev1.Secret) if !ok { - logrus.Errorf("Certificates::handleUpdateEvent: unable to cast object to Secret") + slog.Error("Certificates::handleUpdateEvent: unable to cast object to Secret") return } @@ -194,5 +194,5 @@ func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { c.Certificates[secret.Name][k] = v } - logrus.Debugf("Certificates::handleUpdateEvent: certificates (%d)", len(c.Certificates)) + slog.Debug("Certificates::handleUpdateEvent", slog.Int("certCount", len(c.Certificates))) } diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 2a3c09a..8664e9d 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -8,12 +8,12 @@ package communication import ( "crypto/tls" "fmt" + "log/slog" netHttp "net/http" "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/sirupsen/logrus" ) // NewHTTPClient is a factory method to create a new Http Client with a default configuration. @@ -52,7 +52,7 @@ func NewHeaders(apiKey string) []string { func NewTLSConfig(settings configuration.Settings) *tls.Config { tlsConfig, err := authentication.NewTLSConfig(settings) if err != nil { - logrus.Warnf("Failed to create TLS config: %v", err) + slog.Warn("Failed to create TLS config", "error", err) return &tls.Config{InsecureSkipVerify: true} //nolint:gosec } diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index 91ff27c..ebeacc4 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -8,10 +8,10 @@ package configuration import ( "encoding/base64" "fmt" + "log/slog" "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/sirupsen/logrus" "github.com/spf13/viper" ) @@ -146,7 +146,7 @@ func Read(configName, configPath string) (s Settings, err error) { tlsMode := NoTLS if t, err := validateTLSMode(v.GetString("tls-mode")); err != nil { - logrus.Errorf("could not validate tls mode: %v", err) + slog.Error("could not validate tls mode", "error", err) } else { tlsMode = t } diff --git a/internal/observation/handler.go b/internal/observation/handler.go index bd823f6..9f44359 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -7,12 +7,12 @@ package observation import ( "fmt" + "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" - "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" ) @@ -59,13 +59,13 @@ func NewHandler( // AddRateLimitedEvent adds an event to the event queue func (h *Handler) AddRateLimitedEvent(event *core.Event) { - logrus.Debugf(`Handler::AddRateLimitedEvent: %#v`, event) + slog.Debug(`Handler::AddRateLimitedEvent`, "event", event) h.eventQueue.AddRateLimited(event) } // Run starts the event handler, spins up Goroutines to process events, and waits for a stop signal func (h *Handler) Run(stopCh <-chan struct{}) { - logrus.Debug("Handler::Run") + slog.Debug("Handler::Run") for i := 0; i < h.settings.Handler.Threads; i++ { go wait.Until(h.worker, 0, stopCh) @@ -76,13 +76,13 @@ func (h *Handler) Run(stopCh <-chan struct{}) { // ShutDown stops the event handler and shuts down the event queue func (h *Handler) ShutDown() { - logrus.Debug("Handler::ShutDown") + slog.Debug("Handler::ShutDown") h.eventQueue.ShutDown() } // handleEvent feeds translated events to the synchronizer func (h *Handler) handleEvent(e *core.Event) error { - logrus.Debugf(`Handler::handleEvent: %#v`, e) + slog.Debug("Handler::handleEvent", "event", e) // TODO: Add Telemetry events, err := translation.Translate(e) @@ -97,9 +97,8 @@ func (h *Handler) handleEvent(e *core.Event) error { // handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic func (h *Handler) handleNextEvent() bool { - logrus.Debug("Handler::handleNextEvent") evt, quit := h.eventQueue.Get() - logrus.Debugf(`Handler::handleNextEvent: %#v, quit: %v`, evt, quit) + slog.Debug("Handler::handleNextEvent", "event", evt, "quit", quit) if quit { return false } @@ -121,15 +120,15 @@ func (h *Handler) worker() { // withRetry handles errors from the event handler and requeues events that fail func (h *Handler) withRetry(err error, event *core.Event) { - logrus.Debug("Handler::withRetry") + slog.Debug("Handler::withRetry") if err != nil { // TODO: Add Telemetry if h.eventQueue.NumRequeues(event) < h.settings.Handler.RetryCount { h.eventQueue.AddRateLimited(event) - logrus.Infof(`Handler::withRetry: requeued event: %#v; error: %v`, event, err) + slog.Info("Handler::withRetry: requeued event", "event", event, "error", err) } else { h.eventQueue.Forget(event) - logrus.Warnf(`Handler::withRetry: event %#v has been dropped due to too many retries`, event) + slog.Warn(`Handler::withRetry: event has been dropped due to too many retries`, "event", event) } } // TODO: Add error logging } diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 9cc9a16..c4ea32b 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -9,11 +9,11 @@ import ( "context" "errors" "fmt" + "log/slog" "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -54,7 +54,7 @@ func NewWatcher( // Initialize initializes the Watcher, must be called before Watch func (w *Watcher) Initialize(ctx context.Context) error { - logrus.Debug("Watcher::Initialize") + slog.Debug("Watcher::Initialize") var err error w.informer = w.buildInformer() @@ -70,7 +70,7 @@ func (w *Watcher) Initialize(ctx context.Context) error { // Watch starts the process of watching for changes to Kubernetes resources. // Initialize must be called before Watch. func (w *Watcher) Watch(ctx context.Context) error { - logrus.Debug("Watcher::Watch") + slog.Debug("Watcher::Watch") if w.informer == nil { return errors.New("error: Initialize must be called before Watch") @@ -106,7 +106,7 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { // buildEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) { - logrus.Info("Watcher::buildEventHandlerForAdd") + slog.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { service := obj.(*v1.Service) if !w.isDesiredService(service) { @@ -115,7 +115,7 @@ func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { - logrus.Errorf(`error occurred retrieving node ips: %v`, err) + slog.Error("error occurred retrieving node ips", "error", err) return } @@ -128,7 +128,7 @@ func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) // buildEventHandlerForDelete creates a function that is used as an event handler // for the informer when Delete events are raised. func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface{}) { - logrus.Info("Watcher::buildEventHandlerForDelete") + slog.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { service := obj.(*v1.Service) if !w.isDesiredService(service) { @@ -137,7 +137,7 @@ func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { - logrus.Errorf(`error occurred retrieving node ips: %v`, err) + slog.Error("error occurred retrieving node ips", "error", err) return } @@ -150,7 +150,7 @@ func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface // buildEventHandlerForUpdate creates a function that is used as an event handler // for the informer when Update events are raised. func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface{}, interface{}) { - logrus.Info("Watcher::buildEventHandlerForUpdate") + slog.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API service := updated.(*v1.Service) @@ -160,7 +160,7 @@ func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface nodeIps, err := w.retrieveNodeIps(ctx) if err != nil { - logrus.Errorf(`error occurred retrieving node ips: %v`, err) + slog.Error("error occurred retrieving node ips", "error", err) return } @@ -172,7 +172,7 @@ func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface // buildInformer creates the informer used to watch for changes to Kubernetes resources. func (w *Watcher) buildInformer() cache.SharedIndexInformer { - logrus.Debug("Watcher::buildInformer") + slog.Debug("Watcher::buildInformer") factory := informers.NewSharedInformerFactoryWithOptions( w.k8sClient, w.settings.Watcher.ResyncPeriod, @@ -184,7 +184,7 @@ func (w *Watcher) buildInformer() cache.SharedIndexInformer { // initializeEventListeners initializes the event listeners for the informer. func (w *Watcher) initializeEventListeners(ctx context.Context) error { - logrus.Debug("Watcher::initializeEventListeners") + slog.Debug("Watcher::initializeEventListeners") var err error handlers := cache.ResourceEventHandlerFuncs{ @@ -205,13 +205,13 @@ func (w *Watcher) initializeEventListeners(ctx context.Context) error { // because the master node may or may not be a worker node and thus may not be able to route traffic. func (w *Watcher) retrieveNodeIps(ctx context.Context) ([]string, error) { started := time.Now() - logrus.Debug("Watcher::retrieveNodeIps") + slog.Debug("Watcher::retrieveNodeIps") var nodeIps []string nodes, err := w.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { - logrus.Errorf(`error occurred retrieving the list of nodes: %v`, err) + slog.Error("error occurred retrieving the list of nodes", "error", err) return nil, err } @@ -226,14 +226,14 @@ func (w *Watcher) retrieveNodeIps(ctx context.Context) ([]string, error) { } } - logrus.Debugf("Watcher::retrieveNodeIps duration: %d", time.Since(started).Nanoseconds()) + slog.Debug("Watcher::retrieveNodeIps duration", "duration", time.Since(started).Nanoseconds()) return nodeIps, nil } // notMasterNode determines if the node is a master node. func (w *Watcher) notMasterNode(node v1.Node) bool { - logrus.Debug("Watcher::notMasterNode") + slog.Debug("Watcher::notMasterNode") _, found := node.Labels["node-role.kubernetes.io/master"] diff --git a/internal/probation/server.go b/internal/probation/server.go index 84b5b67..9520fc7 100644 --- a/internal/probation/server.go +++ b/internal/probation/server.go @@ -7,10 +7,9 @@ package probation import ( "fmt" + "log/slog" "net/http" "time" - - "github.com/sirupsen/logrus" ) const ( @@ -27,7 +26,6 @@ const ( // HealthServer is a server that spins up endpoints for the various k8s health checks. type HealthServer struct { - // The underlying HTTP server. httpServer *http.Server @@ -52,7 +50,7 @@ func NewHealthServer() *HealthServer { // Start spins up the health server. func (hs *HealthServer) Start() { - logrus.Debugf("Starting probe listener on port %d", ListenPort) + slog.Debug("Starting probe listener", "port", ListenPort) address := fmt.Sprintf(":%d", ListenPort) @@ -64,17 +62,17 @@ func (hs *HealthServer) Start() { go func() { if err := hs.httpServer.ListenAndServe(); err != nil { - logrus.Errorf("unable to start probe listener on %s: %v", hs.httpServer.Addr, err) + slog.Error("unable to start probe listener", "address", hs.httpServer.Addr, "error", err) } }() - logrus.Info("Started probe listener on", hs.httpServer.Addr) + slog.Info("Started probe listener", "address", hs.httpServer.Addr) } // Stop shuts down the health server. func (hs *HealthServer) Stop() { if err := hs.httpServer.Close(); err != nil { - logrus.Errorf("unable to stop probe listener on %s: %v", hs.httpServer.Addr, err) + slog.Error("unable to stop probe listener", "address", hs.httpServer.Addr, "error", err) } } @@ -99,14 +97,14 @@ func (hs *HealthServer) handleProbe(writer http.ResponseWriter, _ *http.Request, writer.WriteHeader(http.StatusOK) if _, err := fmt.Fprint(writer, Ok); err != nil { - logrus.Error(err) + slog.Error(err.Error()) } } else { writer.WriteHeader(http.StatusServiceUnavailable) if _, err := fmt.Fprint(writer, ServiceNotAvailable); err != nil { - logrus.Error(err) + slog.Error(err.Error()) } } } diff --git a/internal/probation/server_test.go b/internal/probation/server_test.go index 9c7d37d..981aa3b 100644 --- a/internal/probation/server_test.go +++ b/internal/probation/server_test.go @@ -6,11 +6,11 @@ package probation import ( + "log/slog" "net/http" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "github.com/sirupsen/logrus" ) func TestHealthServer_HandleLive(t *testing.T) { @@ -76,5 +76,5 @@ func TestHealthServer_Start(t *testing.T) { t.Errorf("Expected status code %v, got %v", http.StatusAccepted, response.StatusCode) } - logrus.Infof("received a response from the probe server: %v", response) + slog.Info("received a response from the probe server", "response", response) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index eadfbd8..20bc99a 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -7,13 +7,13 @@ package synchronization import ( "fmt" + "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/communication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" nginxClient "github.com/nginxinc/nginx-plus-go-client/client" - "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" ) @@ -58,10 +58,10 @@ func NewSynchronizer( // AddEvents adds a list of events to the queue. If no hosts are specified this is a null operation. // Events will fan out to the number of hosts specified before being added to the queue. func (s *Synchronizer) AddEvents(events core.ServerUpdateEvents) { - logrus.Debugf(`Synchronizer::AddEvents adding %d events`, len(events)) + slog.Debug(`Synchronizer::AddEvents adding events`, slog.Int("eventCount", len(events))) if len(s.settings.NginxPlusHosts) == 0 { - logrus.Warnf(`No Nginx Plus hosts were specified. Skipping synchronization.`) + slog.Warn(`No Nginx Plus hosts were specified. Skipping synchronization.`) return } @@ -75,10 +75,10 @@ func (s *Synchronizer) AddEvents(events core.ServerUpdateEvents) { // AddEvent adds an event to the queue. If no hosts are specified this is a null operation. // Events will be added to the queue after a random delay between MinMillisecondsJitter and MaxMillisecondsJitter. func (s *Synchronizer) AddEvent(event *core.ServerUpdateEvent) { - logrus.Debugf(`Synchronizer::AddEvent: %#v`, event) + slog.Debug(`Synchronizer::AddEvent`, "event", event) if event.NginxHost == `` { - logrus.Warnf(`Nginx host was not specified. Skipping synchronization.`) + slog.Warn(`Nginx host was not specified. Skipping synchronization.`) return } @@ -91,7 +91,7 @@ func (s *Synchronizer) AddEvent(event *core.ServerUpdateEvent) { // Run starts the Synchronizer, spins up Goroutines to process events, and waits for a stop signal. func (s *Synchronizer) Run(stopCh <-chan struct{}) { - logrus.Debug(`Synchronizer::Run`) + slog.Debug(`Synchronizer::Run`) for i := 0; i < s.settings.Synchronizer.Threads; i++ { go wait.Until(s.worker, 0, stopCh) @@ -102,7 +102,7 @@ func (s *Synchronizer) Run(stopCh <-chan struct{}) { // ShutDown stops the Synchronizer and shuts down the event queue func (s *Synchronizer) ShutDown() { - logrus.Debugf(`Synchronizer::ShutDown`) + slog.Debug(`Synchronizer::ShutDown`) s.eventQueue.ShutDownWithDrain() } @@ -110,7 +110,7 @@ func (s *Synchronizer) ShutDown() { // NOTE: There is an open issue (https://github.com/nginxinc/nginx-loadbalancer-kubernetes/issues/36) to move creation // of the underlying Border Server client to the NewBorderClient function. func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (application.Interface, error) { - logrus.Debugf(`Synchronizer::buildBorderClient`) + slog.Debug(`Synchronizer::buildBorderClient`) var err error @@ -129,7 +129,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica // fanOutEventToHosts takes a list of events and returns a list of events, one for each Border Server. func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.ServerUpdateEvents { - logrus.Debugf(`Synchronizer::fanOutEventToHosts: %#v`, event) + slog.Debug(`Synchronizer::fanOutEventToHosts`, "event", event) var events core.ServerUpdateEvents @@ -147,7 +147,7 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se // handleEvent dispatches an event to the proper handler function. func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleEvent: Id: %s`, event.ID) + slog.Debug(`Synchronizer::handleEvent`, slog.String("eventID", event.ID)) var err error @@ -162,13 +162,13 @@ func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { err = s.handleDeletedEvent(event) default: - logrus.Warnf(`Synchronizer::handleEvent: unknown event type: %d`, event.Type) + slog.Warn(`Synchronizer::handleEvent: unknown event type`, "type", event.Type) } if err == nil { - logrus.Infof( - `Synchronizer::handleEvent: successfully %s the nginx+ host(s) for Upstream: %s: Id(%s)`, - event.TypeName(), event.UpstreamName, event.ID) + slog.Info( + "Synchronizer::handleEvent: successfully handled the event", + "type", event.TypeName(), "upstreamName", event.UpstreamName, "eventID", event.ID) } return err @@ -176,7 +176,7 @@ func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { // handleCreatedUpdatedEvent handles events of type Created or Updated. func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleCreatedUpdatedEvent: Id: %s`, serverUpdateEvent.ID) + slog.Debug(`Synchronizer::handleCreatedUpdatedEvent`, "eventID", serverUpdateEvent.ID) var err error @@ -194,7 +194,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU // handleDeletedEvent handles events of type Deleted. func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { - logrus.Debugf(`Synchronizer::handleDeletedEvent: Id: %s`, serverUpdateEvent.ID) + slog.Debug(`Synchronizer::handleDeletedEvent`, "eventID", serverUpdateEvent.ID) var err error @@ -212,7 +212,7 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv // handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic func (s *Synchronizer) handleNextEvent() bool { - logrus.Debug(`Synchronizer::handleNextEvent`) + slog.Debug(`Synchronizer::handleNextEvent`) evt, quit := s.eventQueue.Get() if quit { @@ -229,18 +229,18 @@ func (s *Synchronizer) handleNextEvent() bool { // worker is the main message loop func (s *Synchronizer) worker() { - logrus.Debug(`Synchronizer::worker`) + slog.Debug(`Synchronizer::worker`) for s.handleNextEvent() { } } // withRetry handles errors from the event handler and requeues events that fail func (s *Synchronizer) withRetry(err error, event *core.ServerUpdateEvent) { - logrus.Debug("Synchronizer::withRetry") + slog.Debug("Synchronizer::withRetry") if err != nil { // TODO: Add Telemetry s.eventQueue.AddRateLimited(event) - logrus.Infof(`Synchronizer::withRetry: requeued event: %s; error: %v`, event.ID, err) + slog.Info(`Synchronizer::withRetry: requeued event`, "eventID", event.ID, "error", err) } else { s.eventQueue.Forget(event) } // TODO: Add error logging diff --git a/internal/translation/translator.go b/internal/translation/translator.go index fe8532c..dceecf5 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -7,17 +7,17 @@ package translation import ( "fmt" + "log/slog" "strings" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" ) // Translate transforms event data into an intermediate format that can be consumed by the BorderClient implementations // and used to update the Border Servers. func Translate(event *core.Event) (core.ServerUpdateEvents, error) { - logrus.Debug("Translate::Translate") + slog.Debug("Translate::Translate") return buildServerUpdateEvents(event.Service.Spec.Ports, event) } @@ -29,13 +29,13 @@ func Translate(event *core.Event) (core.ServerUpdateEvents, error) { // The NGINX+ Client uses a single server for Deleted events; // so the list of servers is broken up into individual events. func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.ServerUpdateEvents, error) { - logrus.Debugf("Translate::buildServerUpdateEvents(ports=%#v)", ports) + slog.Debug("Translate::buildServerUpdateEvents", "ports", ports) events := core.ServerUpdateEvents{} for _, port := range ports { context, upstreamName, err := getContextAndUpstreamName(port) if err != nil { - logrus.Info(err) + slog.Info(err.Error()) continue } @@ -56,7 +56,7 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se } default: - logrus.Warnf(`Translator::buildServerUpdateEvents: unknown event type: %d`, event.Type) + slog.Warn(`Translator::buildServerUpdateEvents: unknown event type`, "type", event.Type) } } From 98a140f75f1fd2edbf5d73550fc62adaeba0c56c Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 16 Oct 2024 15:31:59 -0600 Subject: [PATCH 078/136] NLB-6753 Added nlk version to outgoing request headers --- internal/communication/factory.go | 2 ++ internal/communication/factory_test.go | 16 ++++++++++++---- internal/communication/roundtripper_test.go | 10 +++++++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/internal/communication/factory.go b/internal/communication/factory.go index 8664e9d..eec593b 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -14,6 +14,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" ) // NewHTTPClient is a factory method to create a new Http Client with a default configuration. @@ -38,6 +39,7 @@ func NewHeaders(apiKey string) []string { headers := []string{ "Content-Type: application/json", "Accept: application/json", + fmt.Sprintf("X-NLK-Version: %s", buildinfo.SemVer()), } if apiKey != "" { diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index 65f5e5b..7562484 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -31,7 +31,7 @@ func TestNewHeaders(t *testing.T) { t.Fatalf(`headers should not be nil`) } - if len(headers) != 3 { + if len(headers) != 4 { t.Fatalf(`headers should have 3 elements`) } @@ -43,8 +43,12 @@ func TestNewHeaders(t *testing.T) { t.Fatalf(`headers[1] should be "Accept: application/json"`) } - if headers[2] != "Authorization: ApiKey fakeKey" { - t.Fatalf(`headers[2] should be "Accept: Authorization: ApiKey fakeKey"`) + if headers[2] != "X-NLK-Version: " { + t.Fatalf(`headers[2] should be "X-NLK-Version: "`) + } + + if headers[3] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`headers[3] should be "Accept: Authorization: ApiKey fakeKey"`) } } @@ -56,7 +60,7 @@ func TestNewHeadersWithNoAPIKey(t *testing.T) { t.Fatalf(`headers should not be nil`) } - if len(headers) != 2 { + if len(headers) != 3 { t.Fatalf(`headers should have 2 elements`) } @@ -67,6 +71,10 @@ func TestNewHeadersWithNoAPIKey(t *testing.T) { if headers[1] != "Accept: application/json" { t.Fatalf(`headers[1] should be "Accept: application/json"`) } + + if headers[2] != "X-NLK-Version: " { + t.Fatalf(`headers[2] should be "X-NLK-Version: "`) + } } func TestNewTransport(t *testing.T) { diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index 9913600..55dea88 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -28,7 +28,7 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers should not be nil`) } - if len(roundTripper.Headers) != 3 { + if len(roundTripper.Headers) != 4 { t.Fatalf(`roundTripper.Headers should have 3 elements`) } @@ -40,8 +40,12 @@ func TestNewRoundTripper(t *testing.T) { t.Fatalf(`roundTripper.Headers[1] should be "Accept: application/json"`) } - if roundTripper.Headers[2] != "Authorization: ApiKey fakeKey" { - t.Fatalf(`headers[2] should be "Accept: Authorization: ApiKey fakeKey"`) + if roundTripper.Headers[2] != "X-NLK-Version: " { + t.Fatalf(`roundTripper.Headers[2] should be "X-NLK-Version: "`) + } + + if roundTripper.Headers[3] != "Authorization: ApiKey fakeKey" { + t.Fatalf(`roundTripper.Headers[3] should be "Accept: Authorization: ApiKey fakeKey"`) } if roundTripper.RoundTripper == nil { From 58053e8154a5b301509a318047b462886a554cf5 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 16 Oct 2024 15:18:04 -0600 Subject: [PATCH 079/136] NLB-5753 Bumped to version 0.6.1 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index a918a2a..ee6cdce 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.6.0 +0.6.1 From c35132334d4ff56ef0603e0f585222b4c5fd3543 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 7 Nov 2024 01:51:32 +0530 Subject: [PATCH 080/136] NLB-5716: Allow devs to deploy NLK to AKS This commit adds a make target that bootstraps NLK via helm using: - charts from the dev-feature branch. - image from the dev-feature branch. The workflow can be exercised against an AKS cluster and we can interact with NLK via helm post bootstrapping like any user of the controller. --- Makefile | 7 ++++++- scripts/deploy.sh | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100755 scripts/deploy.sh diff --git a/Makefile b/Makefile index 08e07ce..4d00656 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ DOCKER_TAG ?= latest GOPRIVATE = *.f5net.com,gitlab.com/f5 export GOPRIVATE -.PHONY: default tools deps fmt lint test build build.docker publish helm-lint +.PHONY: default tools deps fmt lint test build build.docker publish helm-lint deploy default: build @@ -58,6 +58,11 @@ publish: build-linux build-linux-docker @scripts/docker-login.sh @./scripts/docker.sh publish +deploy: + @[ -f $(KUBECONFIG) ] || (echo "KUBECONFIG not found." && false) + $(MAKE) publish + @scripts/deploy.sh + clean: rm -rf $(BUILD_DIR)/ diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..dc8a56a --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eo pipefail + + +if [ -z "$KUBECONFIG" ]; then + echo "KUBECONFIG is not set." + exit 1 +fi +if [ ! -e "$KUBECONFIG" ]; then + echo "KUBECONFIG does not exist." + exit 1 +fi + +root_dir=$(git rev-parse --show-toplevel) +# shellcheck source=/dev/null +source "${root_dir}/.devops.sh" +devops.backend.docker.set "azure.container-registry-dev" +devops.backend.docker.authenticate + +namespace="nlk" +helm_release_name="release-1" +registry="${DEVOPS_DOCKER_URL}" +repository="nginx-azure-lb/nginxaas-loadbalancer-kubernetes/nginxaas-loadbalancer-kubernetes" +image_tag=$(git rev-parse --short=8 HEAD) + +kubectl create namespace "${namespace}" --dry-run=client -o yaml | kubectl apply -f - +kubectl -n "${namespace}" create secret docker-registry regcred \ + --docker-username="${DEVOPS_DOCKER_USER}" \ + --docker-password="${DEVOPS_DOCKER_PASS}" \ + --docker-server="${DEVOPS_DOCKER_URL}" \ + --dry-run=client -o yaml | kubectl apply -f - + +helm -n "$namespace" upgrade "$helm_release_name" ${root_dir}/charts/nlk/ \ + --set nlk.image.registry="${registry}",nlk.image.repository="${repository}",nlk.image.tag="${image_tag}",nlk.imagePullSecrets[0].name=regcred \ + --install \ + --reuse-values \ + --wait \ + --timeout 2m From 62241befaf49d22624c7538f835b8bdbf98f2ef6 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 7 Nov 2024 02:10:12 +0530 Subject: [PATCH 081/136] Update AKS API version in marketplace bundle The marketplace package needs to have the AKS API as the latest or anything that is under 2 years. Very conservatively choosing the latest version from last year. --- charts/armTemplate.json | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/armTemplate.json b/charts/armTemplate.json index 510c784..06c3703 100644 --- a/charts/armTemplate.json +++ b/charts/armTemplate.json @@ -164,7 +164,7 @@ { "type": "Microsoft.ContainerService/managedClusters", "condition": "[parameters('createNewCluster')]", - "apiVersion": "2022-11-01", + "apiVersion": "2023-11-01", "name": "[parameters('clusterResourceName')]", "location": "[parameters('location')]", "dependsOn": [], diff --git a/version b/version index ee6cdce..b616048 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.6.1 +0.6.2 From b42d27935847588bb140ad4695792276d15643ec Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 7 Nov 2024 12:40:33 -0700 Subject: [PATCH 082/136] NLB-4468 Added package pointer from ARP --- pkg/pointer/pointer.go | 56 +++++++++++++++++++++++++++++++++ pkg/pointer/pointer_test.go | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 pkg/pointer/pointer.go create mode 100644 pkg/pointer/pointer_test.go diff --git a/pkg/pointer/pointer.go b/pkg/pointer/pointer.go new file mode 100644 index 0000000..08ff667 --- /dev/null +++ b/pkg/pointer/pointer.go @@ -0,0 +1,56 @@ +// Package pointer provides utilities that assist in working with pointers. +package pointer + +// To returns a pointer to the given value +func To[T any](v T) *T { return &v } + +// From dereferences the pointer if it is not nil or returns d +func From[T any](p *T, d T) T { + if p != nil { + return *p + } + return d +} + +// ToSlice returns a slice of pointers to the given values. +func ToSlice[T any](values []T) []*T { + if len(values) == 0 { + return nil + } + ret := make([]*T, 0, len(values)) + for _, v := range values { + v := v + ret = append(ret, &v) + } + return ret +} + +// FromSlice returns a slice of values to the given pointers, dropping any nils. +func FromSlice[T any](values []*T) []T { + if len(values) == 0 { + return nil + } + ret := make([]T, 0, len(values)) + for _, v := range values { + if v != nil { + ret = append(ret, *v) + } + } + return ret +} + +// Equal reports if p is a pointer to a value equal to v +func Equal[T comparable](p *T, v T) bool { + if p == nil { + return false + } + return *p == v +} + +// ValueEqual reports if value of pointer referenced by p is equal to value of pointer referenced by q +func ValueEqual[T comparable](p *T, q *T) bool { + if p == nil || q == nil { + return p == q + } + return *p == *q +} diff --git a/pkg/pointer/pointer_test.go b/pkg/pointer/pointer_test.go new file mode 100644 index 0000000..e929e58 --- /dev/null +++ b/pkg/pointer/pointer_test.go @@ -0,0 +1,62 @@ +package pointer_test + +import ( + "testing" + + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/pointer" + "github.com/stretchr/testify/require" +) + +func TestTo(t *testing.T) { + t.Parallel() + + for _, v := range []string{"", "hello"} { + require.Equal(t, v, *pointer.To(v)) + } + for _, v := range []int{0, 123456, -123456} { + require.Equal(t, v, *pointer.To(v)) + } + for _, v := range []int64{0, 123456, -123456} { + require.Equal(t, v, *pointer.To(v)) + } +} + +func TestFrom(t *testing.T) { + t.Parallel() + + sv := "s" + sd := "default" + require.Equal(t, sd, pointer.From(nil, sd)) + require.Equal(t, sv, pointer.From(&sv, sd)) + + iv := 1 + id := 2 + require.Equal(t, id, pointer.From(nil, id)) + require.Equal(t, iv, pointer.From(&iv, id)) + + i64v := int64(1) + i64d := int64(2) + require.Equal(t, i64d, pointer.From(nil, i64d)) + require.Equal(t, i64v, pointer.From(&i64v, i64d)) +} + +func TestToSlice_FromSlice(t *testing.T) { + t.Parallel() + + v := []int{1, 2, 3} + require.Equal(t, v, pointer.FromSlice(pointer.ToSlice(v))) + require.Nil(t, pointer.ToSlice([]string{})) + require.Nil(t, pointer.FromSlice([]*string{})) + require.Equal(t, []string{"A", "B"}, pointer.FromSlice([]*string{pointer.To("A"), nil, pointer.To("B")})) +} + +func TestEqual(t *testing.T) { + t.Parallel() + + require.True(t, pointer.Equal(pointer.To(1), 1)) + require.False(t, pointer.Equal(nil, 1)) + require.False(t, pointer.Equal(pointer.To(1), 2)) + + s := new(struct{}) + require.False(t, pointer.Equal(&s, nil)) +} From a9e2600bd18bf9caaf4bfbcd166b7ce855ee00b1 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 31 Oct 2024 07:13:17 -0600 Subject: [PATCH 083/136] NLB-4468 Added support for cluster IP services, service IP addresses fetched by translator The watcher's role no longer includes sending node IPs to the translator. The watcher simply alerts the translator of a change event with respect to a specific service. It is now the translator's role to determine the upstream server addresses from the nodeport IPs or the cluster IPs, depending on the service type. There were a number of reasons for this change. The main one is that any transitory failure to fetch IP addresses using the kubernetes client will cause the update event to be readded to the workqueue, instead of abandoned. Another benefit is that we can now leverage the existing unit tests for the translator to test the cluster IP functionality here. This commit adds support for cluster IP services. The watcher fetches the service IP addresses and ports from the endpoint slices relevant to the service. The other main change here is to the way that events are handled when a service is deleted. Instead of issuing multiple requests to the nginx hosts to delete specific upstream servers, we now send a single update servers request with an empty list of servers. The nginx plus client extrapolates from the empty list and deletes all existing servers for the upstream. --- cmd/nginx-loadbalancer-kubernetes/main.go | 7 +- go.mod | 2 +- internal/core/event.go | 8 +- internal/core/event_test.go | 7 +- internal/observation/handler.go | 41 +- internal/observation/handler_test.go | 11 +- internal/observation/watcher.go | 83 +- internal/translation/translator.go | 170 ++- internal/translation/translator_test.go | 1489 +++++++++++++++------ test/mocks/mock_handler.go | 15 +- 10 files changed, 1316 insertions(+), 517 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 8064348..00be5d4 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -15,6 +15,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/observation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/probation" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -54,19 +55,19 @@ func run() error { handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue) + handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) watcher, err := observation.NewWatcher(settings, handler, k8sClient) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } - err = watcher.Initialize(ctx) + err = watcher.Initialize() if err != nil { return fmt.Errorf(`error occurred initializing the watcher: %w`, err) } - go handler.Run(ctx.Done()) + go handler.Run(ctx) go synchronizer.Run(ctx.Done()) probeServer := probation.NewHealthServer() diff --git a/go.mod b/go.mod index 94a1319..6b26183 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + golang.org/x/net v0.23.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 @@ -54,7 +55,6 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect diff --git a/internal/core/event.go b/internal/core/event.go index 09776c9..16d5d94 100644 --- a/internal/core/event.go +++ b/internal/core/event.go @@ -24,7 +24,6 @@ const ( // Event represents a service event type Event struct { - // Type represents the event type, one of the constant values defined above. Type EventType @@ -33,18 +32,13 @@ type Event struct { // PreviousService represents the service object in its previous state PreviousService *v1.Service - - // NodeIps represents the list of node IPs in the Cluster. This is populated by the Watcher when an event is created. - // The Node IPs are needed by the BorderClient. - NodeIps []string } // NewEvent factory method to create a new Event -func NewEvent(eventType EventType, service *v1.Service, previousService *v1.Service, nodeIps []string) Event { +func NewEvent(eventType EventType, service *v1.Service, previousService *v1.Service) Event { return Event{ Type: eventType, Service: service, PreviousService: previousService, - NodeIps: nodeIps, } } diff --git a/internal/core/event_test.go b/internal/core/event_test.go index 662eb8f..f0184fb 100644 --- a/internal/core/event_test.go +++ b/internal/core/event_test.go @@ -11,9 +11,8 @@ func TestNewEvent(t *testing.T) { expectedType := Created expectedService := &v1.Service{} expectedPreviousService := &v1.Service{} - expectedNodeIps := []string{"127.0.0.1"} - event := NewEvent(expectedType, expectedService, expectedPreviousService, expectedNodeIps) + event := NewEvent(expectedType, expectedService, expectedPreviousService) if event.Type != expectedType { t.Errorf("expected Created, got %v", event.Type) @@ -26,8 +25,4 @@ func TestNewEvent(t *testing.T) { if event.PreviousService != expectedPreviousService { t.Errorf("expected previous service, got %#v", event.PreviousService) } - - if event.NodeIps[0] != expectedNodeIps[0] { - t.Errorf("expected node ips, got %#v", event.NodeIps) - } } diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 9f44359..2b5bcfb 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -6,13 +6,13 @@ package observation import ( + "context" "fmt" "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" ) @@ -23,7 +23,7 @@ type HandlerInterface interface { AddRateLimitedEvent(event *core.Event) // Run defines the interface used to start the event handler - Run(stopCh <-chan struct{}) + Run(ctx context.Context) // ShutDown defines the interface used to stop the event handler ShutDown() @@ -42,6 +42,12 @@ type Handler struct { // synchronizer is the synchronizer used to synchronize the internal representation with a Border Server synchronizer synchronization.Interface + + translator Translator +} + +type Translator interface { + Translate(context.Context, *core.Event) (core.ServerUpdateEvents, error) } // NewHandler creates a new event handler @@ -49,11 +55,13 @@ func NewHandler( settings configuration.Settings, synchronizer synchronization.Interface, eventQueue workqueue.RateLimitingInterface, + translator Translator, ) *Handler { return &Handler{ eventQueue: eventQueue, settings: settings, synchronizer: synchronizer, + translator: translator, } } @@ -63,15 +71,21 @@ func (h *Handler) AddRateLimitedEvent(event *core.Event) { h.eventQueue.AddRateLimited(event) } -// Run starts the event handler, spins up Goroutines to process events, and waits for a stop signal -func (h *Handler) Run(stopCh <-chan struct{}) { +// Run starts the event handler, spins up Goroutines to process events, and waits for context to be done +func (h *Handler) Run(ctx context.Context) { slog.Debug("Handler::Run") + worker := func() { + for h.handleNextEvent(ctx) { + // TODO: Add Telemetry + } + } + for i := 0; i < h.settings.Handler.Threads; i++ { - go wait.Until(h.worker, 0, stopCh) + go wait.Until(worker, 0, ctx.Done()) } - <-stopCh + <-ctx.Done() } // ShutDown stops the event handler and shuts down the event queue @@ -81,11 +95,11 @@ func (h *Handler) ShutDown() { } // handleEvent feeds translated events to the synchronizer -func (h *Handler) handleEvent(e *core.Event) error { +func (h *Handler) handleEvent(ctx context.Context, e *core.Event) error { slog.Debug("Handler::handleEvent", "event", e) // TODO: Add Telemetry - events, err := translation.Translate(e) + events, err := h.translator.Translate(ctx, e) if err != nil { return fmt.Errorf(`Handler::handleEvent error translating: %v`, err) } @@ -96,7 +110,7 @@ func (h *Handler) handleEvent(e *core.Event) error { } // handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (h *Handler) handleNextEvent() bool { +func (h *Handler) handleNextEvent(ctx context.Context) bool { evt, quit := h.eventQueue.Get() slog.Debug("Handler::handleNextEvent", "event", evt, "quit", quit) if quit { @@ -106,18 +120,11 @@ func (h *Handler) handleNextEvent() bool { defer h.eventQueue.Done(evt) event := evt.(*core.Event) - h.withRetry(h.handleEvent(event), event) + h.withRetry(h.handleEvent(ctx, event), event) return true } -// worker is the main message loop -func (h *Handler) worker() { - for h.handleNextEvent() { - // TODO: Add Telemetry - } -} - // withRetry handles errors from the event handler and requeues events that fail func (h *Handler) withRetry(err error, event *core.Event) { slog.Debug("Handler::withRetry") diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index ba4add1..9dd736c 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -6,6 +6,7 @@ package observation import ( + "context" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" @@ -33,7 +34,7 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { handler.AddRateLimitedEvent(event) - handler.handleNextEvent() + handler.handleNextEvent(context.Background()) if len(synchronizer.Events) != 1 { t.Errorf(`handler.AddRateLimitedEvent did not add the event to the queue`) @@ -46,7 +47,13 @@ func buildHandler() ( eventQueue := &mocks.MockRateLimiter{} synchronizer := &mocks.MockSynchronizer{} - handler := NewHandler(configuration.Settings{}, synchronizer, eventQueue) + handler := NewHandler(configuration.Settings{}, synchronizer, eventQueue, &fakeTranslator{}) return synchronizer, handler } + +type fakeTranslator struct{} + +func (t *fakeTranslator) Translate(ctx context.Context, event *core.Event) (core.ServerUpdateEvents, error) { + return core.ServerUpdateEvents{{}}, nil +} diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index c4ea32b..21ab068 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -10,12 +10,10 @@ import ( "errors" "fmt" "log/slog" - "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -53,13 +51,13 @@ func NewWatcher( } // Initialize initializes the Watcher, must be called before Watch -func (w *Watcher) Initialize(ctx context.Context) error { +func (w *Watcher) Initialize() error { slog.Debug("Watcher::Initialize") var err error w.informer = w.buildInformer() - err = w.initializeEventListeners(ctx) + err = w.initializeEventListeners() if err != nil { return fmt.Errorf(`initialization error: %w`, err) } @@ -105,7 +103,7 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { // buildEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. -func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) { +func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { slog.Info("Watcher::buildEventHandlerForAdd") return func(obj interface{}) { service := obj.(*v1.Service) @@ -113,21 +111,15 @@ func (w *Watcher) buildEventHandlerForAdd(ctx context.Context) func(interface{}) return } - nodeIps, err := w.retrieveNodeIps(ctx) - if err != nil { - slog.Error("error occurred retrieving node ips", "error", err) - return - } - var previousService *v1.Service - e := core.NewEvent(core.Created, service, previousService, nodeIps) + e := core.NewEvent(core.Created, service, previousService) w.handler.AddRateLimitedEvent(&e) } } // buildEventHandlerForDelete creates a function that is used as an event handler // for the informer when Delete events are raised. -func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface{}) { +func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { slog.Info("Watcher::buildEventHandlerForDelete") return func(obj interface{}) { service := obj.(*v1.Service) @@ -135,21 +127,15 @@ func (w *Watcher) buildEventHandlerForDelete(ctx context.Context) func(interface return } - nodeIps, err := w.retrieveNodeIps(ctx) - if err != nil { - slog.Error("error occurred retrieving node ips", "error", err) - return - } - var previousService *v1.Service - e := core.NewEvent(core.Deleted, service, previousService, nodeIps) + e := core.NewEvent(core.Deleted, service, previousService) w.handler.AddRateLimitedEvent(&e) } } // buildEventHandlerForUpdate creates a function that is used as an event handler // for the informer when Update events are raised. -func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface{}, interface{}) { +func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { slog.Info("Watcher::buildEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API @@ -158,14 +144,8 @@ func (w *Watcher) buildEventHandlerForUpdate(ctx context.Context) func(interface return } - nodeIps, err := w.retrieveNodeIps(ctx) - if err != nil { - slog.Error("error occurred retrieving node ips", "error", err) - return - } - previousService := previous.(*v1.Service) - e := core.NewEvent(core.Updated, service, previousService, nodeIps) + e := core.NewEvent(core.Updated, service, previousService) w.handler.AddRateLimitedEvent(&e) } } @@ -183,14 +163,14 @@ func (w *Watcher) buildInformer() cache.SharedIndexInformer { } // initializeEventListeners initializes the event listeners for the informer. -func (w *Watcher) initializeEventListeners(ctx context.Context) error { +func (w *Watcher) initializeEventListeners() error { slog.Debug("Watcher::initializeEventListeners") var err error handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: w.buildEventHandlerForAdd(ctx), - DeleteFunc: w.buildEventHandlerForDelete(ctx), - UpdateFunc: w.buildEventHandlerForUpdate(ctx), + AddFunc: w.buildEventHandlerForAdd(), + DeleteFunc: w.buildEventHandlerForDelete(), + UpdateFunc: w.buildEventHandlerForUpdate(), } w.eventHandlerRegistration, err = w.informer.AddEventHandler(handlers) @@ -200,42 +180,3 @@ func (w *Watcher) initializeEventListeners(ctx context.Context) error { return nil } - -// notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is -// because the master node may or may not be a worker node and thus may not be able to route traffic. -func (w *Watcher) retrieveNodeIps(ctx context.Context) ([]string, error) { - started := time.Now() - slog.Debug("Watcher::retrieveNodeIps") - - var nodeIps []string - - nodes, err := w.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) - if err != nil { - slog.Error("error occurred retrieving the list of nodes", "error", err) - return nil, err - } - - for _, node := range nodes.Items { - // this is kind of a broad assumption, should probably make this a configurable option - if w.notMasterNode(node) { - for _, address := range node.Status.Addresses { - if address.Type == v1.NodeInternalIP { - nodeIps = append(nodeIps, address.Address) - } - } - } - } - - slog.Debug("Watcher::retrieveNodeIps duration", "duration", time.Since(started).Nanoseconds()) - - return nodeIps, nil -} - -// notMasterNode determines if the node is a master node. -func (w *Watcher) notMasterNode(node v1.Node) bool { - slog.Debug("Watcher::notMasterNode") - - _, found := node.Labels["node-role.kubernetes.io/master"] - - return !found -} diff --git a/internal/translation/translator.go b/internal/translation/translator.go index dceecf5..6d7928d 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -6,20 +6,32 @@ package translation import ( + "context" "fmt" "log/slog" "strings" + "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ) +type Translator struct { + k8sClient kubernetes.Interface +} + +func NewTranslator(k8sClient kubernetes.Interface) *Translator { + return &Translator{k8sClient} +} + // Translate transforms event data into an intermediate format that can be consumed by the BorderClient implementations // and used to update the Border Servers. -func Translate(event *core.Event) (core.ServerUpdateEvents, error) { +func (t *Translator) Translate(ctx context.Context, event *core.Event) (core.ServerUpdateEvents, error) { slog.Debug("Translate::Translate") - return buildServerUpdateEvents(event.Service.Spec.Ports, event) + return t.buildServerUpdateEvents(ctx, event.Service.Spec.Ports, event) } // buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type @@ -28,18 +40,108 @@ func Translate(event *core.Event) (core.ServerUpdateEvents, error) { // and the list of servers in NGINX+. // The NGINX+ Client uses a single server for Deleted events; // so the list of servers is broken up into individual events. -func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.ServerUpdateEvents, error) { +func (t *Translator) buildServerUpdateEvents(ctx context.Context, ports []v1.ServicePort, event *core.Event, +) (events core.ServerUpdateEvents, err error) { slog.Debug("Translate::buildServerUpdateEvents", "ports", ports) + switch event.Service.Spec.Type { + case v1.ServiceTypeNodePort: + return t.buildNodeIPEvents(ctx, ports, event) + case v1.ServiceTypeClusterIP: + return t.buildClusterIPEvents(ctx, event) + default: + return events, fmt.Errorf("unsupported service type: %s", event.Service.Spec.Type) + } +} + +type upstream struct { + context string + name string +} + +func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event, +) (events core.ServerUpdateEvents, err error) { + namespace := event.Service.GetObjectMeta().GetNamespace() + serviceName := event.Service.Name + + logger := slog.With("namespace", namespace, "serviceName", serviceName) + logger.Debug("Translate::buildClusterIPEvents") + + if event.Type == core.Deleted { + for _, port := range event.Service.Spec.Ports { + context, upstreamName, pErr := getContextAndUpstreamName(port.Name) + if pErr != nil { + logger.Info(pErr.Error()) + continue + } + events = append(events, core.NewServerUpdateEvent(core.Updated, upstreamName, context, nil)) + } + return events, nil + } + + s := t.k8sClient.DiscoveryV1().EndpointSlices(namespace) + list, err := s.List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("kubernetes.io/service-name=%s", serviceName)}) + if err != nil { + logger.Error(`error occurred retrieving the list of endpoint slices`, "error", err) + return events, err + } + + upstreams := make(map[upstream][]*core.UpstreamServer) + + for _, endpointSlice := range list.Items { + for _, port := range endpointSlice.Ports { + if port.Name == nil || port.Port == nil { + continue + } + + context, upstreamName, err := getContextAndUpstreamName(*port.Name) + if err != nil { + logger.Info(err.Error()) + continue + } + + u := upstream{ + context: context, + name: upstreamName, + } + servers := upstreams[u] + + for _, endpoint := range endpointSlice.Endpoints { + for _, address := range endpoint.Addresses { + host := fmt.Sprintf("%s:%d", address, *port.Port) + servers = append(servers, core.NewUpstreamServer(host)) + } + } + + upstreams[u] = servers + } + } + + for u, servers := range upstreams { + events = append(events, core.NewServerUpdateEvent(core.Updated, u.name, u.context, servers)) + } + + return events, nil +} + +func (t *Translator) buildNodeIPEvents(ctx context.Context, ports []v1.ServicePort, event *core.Event, +) (core.ServerUpdateEvents, error) { + slog.Debug("Translate::buildNodeIPEvents", "ports", ports) + events := core.ServerUpdateEvents{} for _, port := range ports { - context, upstreamName, err := getContextAndUpstreamName(port) + context, upstreamName, err := getContextAndUpstreamName(port.Name) if err != nil { slog.Info(err.Error()) continue } - upstreamServers := buildUpstreamServers(event.NodeIps, port) + addresses, err := t.retrieveNodeIps(ctx) + if err != nil { + return nil, err + } + + upstreamServers := buildUpstreamServers(addresses, port) switch event.Type { case core.Created: @@ -49,14 +151,11 @@ func buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event) (core.Se events = append(events, core.NewServerUpdateEvent(event.Type, upstreamName, context, upstreamServers)) case core.Deleted: - for _, server := range upstreamServers { - events = append(events, core.NewServerUpdateEvent( - event.Type, upstreamName, context, core.UpstreamServers{server}, - )) - } - + events = append(events, core.NewServerUpdateEvent( + core.Updated, upstreamName, context, nil, + )) default: - slog.Warn(`Translator::buildServerUpdateEvents: unknown event type`, "type", event.Type) + slog.Warn(`Translator::buildNodeIPEvents: unknown event type`, "type", event.Type) } } @@ -78,15 +177,54 @@ func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamSe // getContextAndUpstreamName returns the nginx context being supplied by the port (either "http" or "stream") // and the upstream name. -func getContextAndUpstreamName(port v1.ServicePort) (clientType string, appName string, err error) { - context, upstreamName, found := strings.Cut(port.Name, "-") +func getContextAndUpstreamName(portName string) (clientType string, appName string, err error) { + context, upstreamName, found := strings.Cut(portName, "-") switch { case !found: return clientType, appName, - fmt.Errorf("ignoring port %s because it is not in the format [http|stream]-{upstreamName}", port.Name) + fmt.Errorf("ignoring port %s because it is not in the format [http|stream]-{upstreamName}", portName) case context != "http" && context != "stream": - return clientType, appName, fmt.Errorf("port name %s does not include \"http\" or \"stream\" context", port.Name) + return clientType, appName, fmt.Errorf("port name %s does not include \"http\" or \"stream\" context", portName) default: return context, upstreamName, nil } } + +// notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is +// because the master node may or may not be a worker node and thus may not be able to route traffic. +func (t *Translator) retrieveNodeIps(ctx context.Context) ([]string, error) { + started := time.Now() + slog.Debug("Translator::retrieveNodeIps") + + var nodeIps []string + + nodes, err := t.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + slog.Error("error occurred retrieving the list of nodes", "error", err) + return nil, err + } + + for _, node := range nodes.Items { + // this is kind of a broad assumption, should probably make this a configurable option + if notMasterNode(node) { + for _, address := range node.Status.Addresses { + if address.Type == v1.NodeInternalIP { + nodeIps = append(nodeIps, address.Address) + } + } + } + } + + slog.Debug("Translator::retrieveNodeIps duration", "duration", time.Since(started).Nanoseconds()) + + return nodeIps, nil +} + +// notMasterNode determines if the node is a master node. +func notMasterNode(node v1.Node) bool { + slog.Debug("Translator::notMasterNode") + + _, found := node.Labels["node-role.kubernetes.io/master"] + + return !found +} diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index 5b508c3..73c99a4 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -12,7 +12,12 @@ import ( "time" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" + "github.com/nginxinc/kubernetes-nginx-ingress/pkg/pointer" + "golang.org/x/net/context" v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" ) const ( @@ -20,6 +25,9 @@ const ( ManyNodes = 7 NoNodes = 0 OneNode = 1 + ManyEndpointSlices = 7 + NoEndpointSlices = 0 + OneEndpointSlice = 1 TranslateErrorFormat = "Translate() error = %v" ) @@ -29,130 +37,266 @@ const ( func TestCreatedTranslateNoPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 0 + testcases := map[string]struct{ serviceType v1.ServiceType }{ + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + } - service := defaultService() - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildCreatedEvent(service) + + translator := NewTranslator(NewFakeClient([]discovery.EndpointSlice{}, []v1.Node{})) - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } func TestCreatedTranslateNoInterestingPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 + testcases := map[string]struct{ serviceType v1.ServiceType }{ + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + } - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 0 + const portCount = 1 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) + + translator := NewTranslator(NewFakeClient([]discovery.EndpointSlice{}, []v1.Node{})) - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } +//nolint:dupl func TestCreatedTranslateOneInterestingPort(t *testing.T) { t.Parallel() - const expectedEventCount = 1 - const portCount = 1 + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), + expectedServerCount: OneEndpointSlice, + }, + } - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 1 + const portCount = 1 - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) - assertExpectedServerCount(t, OneNode, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) + } } +//nolint:dupl func TestCreatedTranslateManyInterestingPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 4 - const portCount = 4 + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), + expectedServerCount: OneEndpointSlice, + }, + } - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 4 + const portCount = 4 - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) - assertExpectedServerCount(t, OneNode, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) + } } +//nolint:dupl func TestCreatedTranslateManyMixedPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) - assertExpectedServerCount(t, OneNode, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) + } } func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildCreatedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + expectedServerCount: ManyNodes, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), + expectedServerCount: ManyEndpointSlices, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildCreatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, ManyNodes, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } /* @@ -161,130 +305,289 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { func TestUpdatedTranslateNoPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - service := defaultService() - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 - - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + const portCount = 1 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + }) } } func TestUpdatedTranslateOneInterestingPort(t *testing.T) { t.Parallel() - const expectedEventCount = 1 - const portCount = 1 - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 1 + const portCount = 1 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, OneNode, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 4 - const portCount = 4 - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 4 + const portCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestUpdatedTranslateManyMixedPorts(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + expectedServerCount: OneNode, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), + expectedServerCount: OneEndpointSlice, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildUpdatedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + expectedServerCount int + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + expectedServerCount: ManyNodes, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), + expectedServerCount: ManyEndpointSlices, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 2 + const portCount = 6 + const updatablePortCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildUpdatedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, tc.expectedServerCount, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } /* @@ -293,331 +596,682 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - service := defaultService() - event := buildDeletedEvent(service, NoNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + const portCount = 1 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } +//nolint:dupl func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(0, 1, 1), + }, + } - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) - } + const expectedEventCount = 1 + const portCount = 1 - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 4 - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(0, 4, 4), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 4 + const expectedEventCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, NoNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(0, 6, 2), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const portCount = 6 + const updatablePortCount = 2 + const expectedEventCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) - assertExpectedServerCount(t, ManyNodes, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - - service := defaultService() - event := buildDeletedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildDeletedEvent(service) - assertExpectedServerCount(t, ManyNodes, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - const portCount = 1 - - ports := generateUpdatablePorts(portCount, 0) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 1 + const expectedEventCount = 0 + + ports := generateUpdatablePorts(portCount, 0) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } +//nolint:dupl func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 1 - const portCount = 1 - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const portCount = 1 + const expectedEventCount = 1 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } - assertExpectedServerCount(t, OneNode, translatedEvents) + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } +//nolint:dupl func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 4 - const portCount = 4 - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + + const portCount = 4 + const expectedEventCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) - assertExpectedServerCount(t, OneNode, translatedEvents) + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { t.Parallel() - const expectedEventCount = 2 - const portCount = 6 - const updatablePortCount = 2 - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, OneNode) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(OneNode), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 6 + const updatablePortCount = 2 + const expectedEventCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { t.Parallel() - const expectedEventCount = 0 - service := defaultService() - event := buildDeletedEvent(service, ManyNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 0, 0), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) - } + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const expectedEventCount = 0 + + service := defaultService(tc.serviceType) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } - assertExpectedServerCount(t, ManyNodes, translatedEvents) + assertExpectedServerCount(t, 0, translatedEvents) + }) + } } func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { t.Parallel() - const portCount = 1 - const updatablePortCount = 0 - const expectedEventCount = updatablePortCount * ManyNodes - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 0), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 1 + const updatablePortCount = 0 + const expectedEventCount = updatablePortCount * ManyNodes + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, ManyNodes, translatedEvents) } +//nolint:dupl func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { t.Parallel() - const portCount = 1 - const expectedEventCount = portCount * ManyNodes - - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 1), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 1 + const expectedEventCount = 1 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } +//nolint:dupl func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { t.Parallel() - const portCount = 4 - const expectedEventCount = portCount * ManyNodes - ports := generatePorts(portCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 4, 4), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 4 + const expectedEventCount = 4 + + ports := generatePorts(portCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { t.Parallel() - const portCount = 6 - const updatablePortCount = 2 - const expectedEventCount = updatablePortCount * ManyNodes - - ports := generateUpdatablePorts(portCount, updatablePortCount) - service := serviceWithPorts(ports) - event := buildDeletedEvent(service, ManyNodes) - translatedEvents, err := Translate(&event) - if err != nil { - t.Fatalf(TranslateErrorFormat, err) + testcases := map[string]struct { + serviceType v1.ServiceType + nodes []v1.Node + endpoints []discovery.EndpointSlice + }{ + "nodePort": { + serviceType: v1.ServiceTypeNodePort, + nodes: generateNodes(ManyNodes), + }, + "clusterIP": { + serviceType: v1.ServiceTypeClusterIP, + endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), + }, } - actualEventCount := len(translatedEvents) - if actualEventCount != expectedEventCount { - t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + for name, tc := range testcases { + tc := tc + t.Run(name, func(t *testing.T) { + t.Parallel() + const portCount = 6 + const updatablePortCount = 2 + const expectedEventCount = 2 + + ports := generateUpdatablePorts(portCount, updatablePortCount) + service := serviceWithPorts(tc.serviceType, ports) + event := buildDeletedEvent(service) + + translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) + translatedEvents, err := translator.Translate(context.TODO(), &event) + if err != nil { + t.Fatalf(TranslateErrorFormat, err) + } + + actualEventCount := len(translatedEvents) + if actualEventCount != expectedEventCount { + t.Fatalf(AssertionFailureFormat, expectedEventCount, actualEventCount) + } + + assertExpectedServerCount(t, 0, translatedEvents) + }) } - - assertExpectedServerCount(t, OneNode, translatedEvents) } func assertExpectedServerCount(t *testing.T, expectedCount int, events core.ServerUpdateEvents) { @@ -629,46 +1283,98 @@ func assertExpectedServerCount(t *testing.T, expectedCount int, events core.Serv } } -func defaultService() *v1.Service { - return &v1.Service{} +func defaultService(serviceType v1.ServiceType) *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-service", + Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, + }, + Spec: v1.ServiceSpec{ + Type: serviceType, + }, + } } -func serviceWithPorts(ports []v1.ServicePort) *v1.Service { +func serviceWithPorts(serviceType v1.ServiceType, ports []v1.ServicePort) *v1.Service { return &v1.Service{ Spec: v1.ServiceSpec{ + Type: serviceType, Ports: ports, }, } } -func buildCreatedEvent(service *v1.Service, nodeCount int) core.Event { - return buildEvent(core.Created, service, nodeCount) +func buildCreatedEvent(service *v1.Service) core.Event { + return buildEvent(core.Created, service) } -func buildDeletedEvent(service *v1.Service, nodeCount int) core.Event { - return buildEvent(core.Deleted, service, nodeCount) +func buildDeletedEvent(service *v1.Service) core.Event { + return buildEvent(core.Deleted, service) } -func buildUpdatedEvent(service *v1.Service, nodeCount int) core.Event { - return buildEvent(core.Updated, service, nodeCount) +func buildUpdatedEvent(service *v1.Service) core.Event { + return buildEvent(core.Updated, service) } -func buildEvent(eventType core.EventType, service *v1.Service, nodeCount int) core.Event { - previousService := defaultService() +func buildEvent(eventType core.EventType, service *v1.Service) core.Event { + previousService := defaultService(service.Spec.Type) - nodeIps := generateNodeIps(nodeCount) + event := core.NewEvent(eventType, service, previousService) + event.Service.Name = "default-service" + return event +} - return core.NewEvent(eventType, service, previousService, nodeIps) +func generateNodes(count int) (nodes []v1.Node) { + for i := 0; i < count; i++ { + nodes = append(nodes, v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("node%d", i), + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: fmt.Sprintf("10.0.0.%v", i), + }, + }, + }, + }) + } + + return nodes } -func generateNodeIps(count int) []string { - var nodeIps []string +func generateEndpointSlices(endpointCount, portCount, updatablePortCount int, +) (endpointSlices []discovery.EndpointSlice) { + servicePorts := generateUpdatablePorts(portCount, updatablePortCount) - for i := 0; i < count; i++ { - nodeIps = append(nodeIps, fmt.Sprintf("10.0.0.%v", i)) + ports := make([]discovery.EndpointPort, 0, len(servicePorts)) + for _, servicePort := range servicePorts { + ports = append(ports, discovery.EndpointPort{ + Name: pointer.To(servicePort.Name), + Port: pointer.To(int32(8080)), + }) + } + + var endpoints []discovery.Endpoint + for i := 0; i < endpointCount; i++ { + endpoints = append(endpoints, discovery.Endpoint{ + Addresses: []string{ + fmt.Sprintf("10.0.0.%v", i), + }, + }) } - return nodeIps + endpointSlices = append(endpointSlices, discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "endpointSlice", + Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, + }, + Endpoints: endpoints, + Ports: ports, + }) + + return endpointSlices } func generatePorts(portCount int) []v1.ServicePort { @@ -708,3 +1414,14 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort return ports } + +func NewFakeClient(endpointSlices []discovery.EndpointSlice, nodes []v1.Node) *fake.Clientset { + return fake.NewSimpleClientset( + &discovery.EndpointSliceList{ + Items: endpointSlices, + }, + &v1.NodeList{ + Items: nodes, + }, + ) +} diff --git a/test/mocks/mock_handler.go b/test/mocks/mock_handler.go index b854db9..f144cea 100644 --- a/test/mocks/mock_handler.go +++ b/test/mocks/mock_handler.go @@ -5,23 +5,22 @@ package mocks -import "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" +import ( + "context" -type MockHandler struct { -} + "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" +) -func (h *MockHandler) AddRateLimitedEvent(_ *core.Event) { +type MockHandler struct{} +func (h *MockHandler) AddRateLimitedEvent(_ *core.Event) { } func (h *MockHandler) Initialize() { - } -func (h *MockHandler) Run(_ <-chan struct{}) { - +func (h *MockHandler) Run(ctx context.Context) { } func (h *MockHandler) ShutDown() { - } From 53927cb0c9e6b7d268f43e39e3c47d55723ed723 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 7 Nov 2024 14:58:10 -0700 Subject: [PATCH 084/136] NLB-4468 Bumped version to 0.7.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index b616048..faef31a 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.6.2 +0.7.0 From 57ab4941bb466313e373f35b712427581174f9cb Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 8 Nov 2024 02:02:38 +0530 Subject: [PATCH 085/136] NLB-5868: Use same image tag as appVersion We have always aimed at keeping the chart and app versions the same, but we also set the image tag in the values file. Doing so forces the image tag to be set to what's in the values files instead of using the appVersion of the Chart. This leads to issues where a newer version of the chart has an older image tag. What we want instead is to use the appVersion of the Helm chart as the image tag and only use the image tag in the values file when it is set. --- charts/nlk/values.yaml | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index f5be244..851f67d 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -18,7 +18,7 @@ nlk: repository: nginx/nginxaas-loadbalancer-kubernetes pullPolicy: Always ## Overrides the image tag whose default is the chart appVersion. - tag: 0.4.0 + # tag: 0.4.0 imagePullSecrets: [] nameOverride: "" fullnameOverride: "" diff --git a/version b/version index faef31a..39e898a 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.7.0 +0.7.1 From 5b5689d01c583b8a55ed1517f12aa82f1813473a Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 11 Nov 2024 09:13:04 -0700 Subject: [PATCH 086/136] NLB-5872 NLK main routine uses a shared informer factory All NLK modules can now consume shared informer resources that are created from a single factory. In this commit the only consumer of this shared factory is the watcher, but in future the translator will make use of it as well. The pattern makes initializing shared resources easier (e.g. a single call to wait for sync, a single call to start the factory). It will also help to avoid circular dependencies as modules will not be dependent on each other to expose shared informer cache resources. I've followed patterns laid down by the kubernetes project's sample controller with respect to the watcher. The watcher constructor now adds event handler routines to its shared informer before the informers are started. This eliminates the need for a separate Initialize() routine. --- cmd/nginx-loadbalancer-kubernetes/main.go | 20 ++-- internal/observation/watcher.go | 110 ++++++++-------------- internal/observation/watcher_test.go | 17 ++-- 3 files changed, 61 insertions(+), 86 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 00be5d4..99240d3 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -17,6 +17,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" @@ -53,18 +54,25 @@ func run() error { return fmt.Errorf(`error initializing synchronizer: %w`, err) } + factory := informers.NewSharedInformerFactoryWithOptions( + k8sClient, settings.Watcher.ResyncPeriod, + ) + handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) - watcher, err := observation.NewWatcher(settings, handler, k8sClient) + watcher, err := observation.NewWatcher(settings, handler, factory.Core().V1().Services()) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } - err = watcher.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing the watcher: %w`, err) + factory.Start(ctx.Done()) + results := factory.WaitForCacheSync(ctx.Done()) + for name, success := range results { + if !success { + return fmt.Errorf(`error occurred waiting for cache sync for %s`, name) + } } go handler.Run(ctx) @@ -73,9 +81,9 @@ func run() error { probeServer := probation.NewHealthServer() probeServer.Start() - err = watcher.Watch(ctx) + err = watcher.Run(ctx) if err != nil { - return fmt.Errorf(`error occurred watching for events: %w`, err) + return fmt.Errorf(`error occurred running watcher: %w`, err) } <-ctx.Done() diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 21ab068..5a2905e 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -7,7 +7,6 @@ package observation import ( "context" - "errors" "fmt" "log/slog" @@ -15,8 +14,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" + coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/tools/cache" ) @@ -24,69 +22,53 @@ import ( // Particularly, Services in the namespace defined in the WatcherSettings::NginxIngressNamespace setting. // When a change is detected, an Event is generated and added to the Handler's queue. type Watcher struct { - // eventHandlerRegistration is used to track the event handlers - eventHandlerRegistration interface{} - // handler is the event handler handler HandlerInterface - // informer is the informer used to watch for changes to Kubernetes resources - informer cache.SharedIndexInformer - - k8sClient kubernetes.Interface - // settings is the configuration settings settings configuration.Settings + + // servicesInformer is the informer used to watch for changes to services + servicesInformer cache.SharedIndexInformer } // NewWatcher creates a new Watcher func NewWatcher( - settings configuration.Settings, handler HandlerInterface, k8sClient kubernetes.Interface, + settings configuration.Settings, + handler HandlerInterface, + serviceInformer coreinformers.ServiceInformer, ) (*Watcher, error) { - return &Watcher{ - handler: handler, - settings: settings, - k8sClient: k8sClient, - }, nil -} + if serviceInformer == nil { + return nil, fmt.Errorf("service informer cannot be nil") + } -// Initialize initializes the Watcher, must be called before Watch -func (w *Watcher) Initialize() error { - slog.Debug("Watcher::Initialize") - var err error + servicesInformer := serviceInformer.Informer() - w.informer = w.buildInformer() + w := &Watcher{ + handler: handler, + settings: settings, + servicesInformer: servicesInformer, + } - err = w.initializeEventListeners() - if err != nil { - return fmt.Errorf(`initialization error: %w`, err) + if err := w.initializeEventListeners(servicesInformer); err != nil { + return nil, err } - return nil + return w, nil } -// Watch starts the process of watching for changes to Kubernetes resources. +// Run starts the process of watching for changes to Kubernetes resources. // Initialize must be called before Watch. -func (w *Watcher) Watch(ctx context.Context) error { - slog.Debug("Watcher::Watch") - - if w.informer == nil { - return errors.New("error: Initialize must be called before Watch") +func (w *Watcher) Run(ctx context.Context) error { + if w.servicesInformer == nil { + return fmt.Errorf(`servicesInformer is nil`) } + slog.Debug("Watcher::Watch") + defer utilruntime.HandleCrash() defer w.handler.ShutDown() - go w.informer.Run(ctx.Done()) - - if !cache.WaitForNamedCacheSync( - w.settings.Handler.WorkQueueSettings.Name, - ctx.Done(), - w.informer.HasSynced, - ) { - return fmt.Errorf(`error occurred waiting for the cache to sync`) - } - <-ctx.Done() return nil } @@ -101,10 +83,10 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { return annotation == w.settings.Watcher.ServiceAnnotation } -// buildEventHandlerForAdd creates a function that is used as an event handler +// buildServiceEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. -func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { - slog.Info("Watcher::buildEventHandlerForAdd") +func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { + slog.Info("Watcher::buildServiceEventHandlerForAdd") return func(obj interface{}) { service := obj.(*v1.Service) if !w.isDesiredService(service) { @@ -117,10 +99,10 @@ func (w *Watcher) buildEventHandlerForAdd() func(interface{}) { } } -// buildEventHandlerForDelete creates a function that is used as an event handler +// buildServiceEventHandlerForDelete creates a function that is used as an event handler // for the informer when Delete events are raised. -func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { - slog.Info("Watcher::buildEventHandlerForDelete") +func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { + slog.Info("Watcher::buildServiceEventHandlerForDelete") return func(obj interface{}) { service := obj.(*v1.Service) if !w.isDesiredService(service) { @@ -133,10 +115,10 @@ func (w *Watcher) buildEventHandlerForDelete() func(interface{}) { } } -// buildEventHandlerForUpdate creates a function that is used as an event handler +// buildServiceEventHandlerForUpdate creates a function that is used as an event handler // for the informer when Update events are raised. -func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { - slog.Info("Watcher::buildEventHandlerForUpdate") +func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interface{}) { + slog.Info("Watcher::buildServiceEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API service := updated.(*v1.Service) @@ -150,30 +132,20 @@ func (w *Watcher) buildEventHandlerForUpdate() func(interface{}, interface{}) { } } -// buildInformer creates the informer used to watch for changes to Kubernetes resources. -func (w *Watcher) buildInformer() cache.SharedIndexInformer { - slog.Debug("Watcher::buildInformer") - - factory := informers.NewSharedInformerFactoryWithOptions( - w.k8sClient, w.settings.Watcher.ResyncPeriod, - ) - informer := factory.Core().V1().Services().Informer() - - return informer -} - // initializeEventListeners initializes the event listeners for the informer. -func (w *Watcher) initializeEventListeners() error { +func (w *Watcher) initializeEventListeners( + servicesInformer cache.SharedIndexInformer, +) error { slog.Debug("Watcher::initializeEventListeners") var err error handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: w.buildEventHandlerForAdd(), - DeleteFunc: w.buildEventHandlerForDelete(), - UpdateFunc: w.buildEventHandlerForUpdate(), + AddFunc: w.buildServiceEventHandlerForAdd(), + DeleteFunc: w.buildServiceEventHandlerForDelete(), + UpdateFunc: w.buildServiceEventHandlerForUpdate(), } - w.eventHandlerRegistration, err = w.informer.AddEventHandler(handlers) + _, err = servicesInformer.AddEventHandler(handlers) if err != nil { return fmt.Errorf(`error occurred adding event handlers: %w`, err) } diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index f8de849..6c78297 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -6,25 +6,20 @@ package observation import ( - "context" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - "k8s.io/client-go/kubernetes" + "github.com/stretchr/testify/require" ) -func TestWatcher_MustInitialize(t *testing.T) { +func TestWatcher_ErrWithNilInformer(t *testing.T) { t.Parallel() - watcher, _ := buildWatcher() - if err := watcher.Watch(context.Background()); err == nil { - t.Errorf("Expected error, got %s", err) - } + _, err := buildWatcherWithNilInformer() + require.Error(t, err, "expected construction of watcher with nil informer to fail") } -func buildWatcher() (*Watcher, error) { - k8sClient := &kubernetes.Clientset{} +func buildWatcherWithNilInformer() (*Watcher, error) { handler := &mocks.MockHandler{} - - return NewWatcher(configuration.Settings{}, handler, k8sClient) + return NewWatcher(configuration.Settings{}, handler, nil) } From a8dd150b084fb47e42d48663bf8a2fc75975921a Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 13 Nov 2024 15:07:27 -0700 Subject: [PATCH 087/136] NLB-5872 Added event handlers for endpoint slice events Also added the register of services we care about --- cmd/nginx-loadbalancer-kubernetes/main.go | 5 +- internal/observation/register.go | 49 ++++++++++ internal/observation/watcher.go | 107 ++++++++++++++++++++-- internal/observation/watcher_test.go | 2 +- 4 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 internal/observation/register.go diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 99240d3..21c609c 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -58,11 +58,14 @@ func run() error { k8sClient, settings.Watcher.ResyncPeriod, ) + serviceInformer := factory.Core().V1().Services() + endpointSliceInformer := factory.Discovery().V1().EndpointSlices() + handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) - watcher, err := observation.NewWatcher(settings, handler, factory.Core().V1().Services()) + watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } diff --git a/internal/observation/register.go b/internal/observation/register.go new file mode 100644 index 0000000..26ab86d --- /dev/null +++ b/internal/observation/register.go @@ -0,0 +1,49 @@ +package observation + +import ( + "sync" + + v1 "k8s.io/api/core/v1" +) + +// register holds references to the services that the user has configured for use with NLK +type register struct { + mu sync.RWMutex // protects register + services map[registerKey]*v1.Service +} + +type registerKey struct { + serviceName string + namespace string +} + +func newRegister() *register { + return ®ister{ + services: make(map[registerKey]*v1.Service), + } +} + +// addOrUpdateService adds the service to the register if not found, else updates the existing service +func (r *register) addOrUpdateService(service *v1.Service) { + r.mu.Lock() + defer r.mu.Unlock() + + r.services[registerKey{namespace: service.Namespace, serviceName: service.Name}] = service +} + +// removeService removes the service from the register +func (r *register) removeService(service *v1.Service) { + r.mu.Lock() + defer r.mu.Unlock() + + delete(r.services, registerKey{namespace: service.Namespace, serviceName: service.Name}) +} + +// getService returns the service from the register if found +func (r *register) getService(namespace string, serviceName string) (*v1.Service, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + s, ok := r.services[registerKey{namespace: namespace, serviceName: serviceName}] + return s, ok +} diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 5a2905e..9752b38 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -13,8 +13,10 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" coreinformers "k8s.io/client-go/informers/core/v1" + discoveryinformers "k8s.io/client-go/informers/discovery/v1" "k8s.io/client-go/tools/cache" ) @@ -30,6 +32,11 @@ type Watcher struct { // servicesInformer is the informer used to watch for changes to services servicesInformer cache.SharedIndexInformer + + // endpointSliceInformer is the informer used to watch for changes to endpoint slices + endpointSliceInformer cache.SharedIndexInformer + + register *register } // NewWatcher creates a new Watcher @@ -37,17 +44,25 @@ func NewWatcher( settings configuration.Settings, handler HandlerInterface, serviceInformer coreinformers.ServiceInformer, + endpointSliceInformer discoveryinformers.EndpointSliceInformer, ) (*Watcher, error) { if serviceInformer == nil { return nil, fmt.Errorf("service informer cannot be nil") } + if endpointSliceInformer == nil { + return nil, fmt.Errorf("endpoint slice informer cannot be nil") + } + servicesInformer := serviceInformer.Informer() + endpointSlicesInformer := endpointSliceInformer.Informer() w := &Watcher{ - handler: handler, - settings: settings, - servicesInformer: servicesInformer, + handler: handler, + settings: settings, + servicesInformer: servicesInformer, + endpointSliceInformer: endpointSlicesInformer, + register: newRegister(), } if err := w.initializeEventListeners(servicesInformer); err != nil { @@ -83,6 +98,69 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { return annotation == w.settings.Watcher.ServiceAnnotation } +func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { + slog.Info("Watcher::buildEndpointSlicesEventHandlerForAdd") + return func(obj interface{}) { + endpointSlice, ok := obj.(*discovery.EndpointSlice) + if !ok { + slog.Error("could not convert event object to EndpointSlice", "obj", obj) + return + } + + service, ok := w.register.getService(endpointSlice.Namespace, endpointSlice.Labels["kubernetes.io/service-name"]) + if !ok { + // not interested in any unregistered service + return + } + + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } +} + +func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, interface{}) { + slog.Info("Watcher::buildEndpointSlicesEventHandlerForUpdate") + return func(previous, updated interface{}) { + endpointSlice, ok := updated.(*discovery.EndpointSlice) + if !ok { + slog.Error("could not convert event object to EndpointSlice", "obj", updated) + return + } + + service, ok := w.register.getService(endpointSlice.Namespace, endpointSlice.Labels["kubernetes.io/service-name"]) + if !ok { + // not interested in any unregistered service + return + } + + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } +} + +func (w *Watcher) buildEndpointSlicesEventHandlerForDelete() func(interface{}) { + slog.Info("Watcher::buildEndpointSlicesEventHandlerForDelete") + return func(obj interface{}) { + endpointSlice, ok := obj.(*discovery.EndpointSlice) + if !ok { + slog.Error("could not convert event object to EndpointSlice", "obj", obj) + return + } + + service, ok := w.register.getService(endpointSlice.Namespace, endpointSlice.Labels["kubernetes.io/service-name"]) + if !ok { + // not interested in any unregistered service + return + } + + var previousService *v1.Service + e := core.NewEvent(core.Deleted, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } +} + // buildServiceEventHandlerForAdd creates a function that is used as an event handler // for the informer when Add events are raised. func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { @@ -93,6 +171,8 @@ func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { return } + w.register.addOrUpdateService(service) + var previousService *v1.Service e := core.NewEvent(core.Created, service, previousService) w.handler.AddRateLimitedEvent(&e) @@ -109,6 +189,8 @@ func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { return } + w.register.removeService(service) + var previousService *v1.Service e := core.NewEvent(core.Deleted, service, previousService) w.handler.AddRateLimitedEvent(&e) @@ -126,6 +208,8 @@ func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interfac return } + w.register.addOrUpdateService(service) + previousService := previous.(*v1.Service) e := core.NewEvent(core.Updated, service, previousService) w.handler.AddRateLimitedEvent(&e) @@ -139,15 +223,26 @@ func (w *Watcher) initializeEventListeners( slog.Debug("Watcher::initializeEventListeners") var err error - handlers := cache.ResourceEventHandlerFuncs{ + serviceHandlers := cache.ResourceEventHandlerFuncs{ AddFunc: w.buildServiceEventHandlerForAdd(), DeleteFunc: w.buildServiceEventHandlerForDelete(), UpdateFunc: w.buildServiceEventHandlerForUpdate(), } - _, err = servicesInformer.AddEventHandler(handlers) + endpointSliceHandlers := cache.ResourceEventHandlerFuncs{ + AddFunc: w.buildEndpointSlicesEventHandlerForAdd(), + DeleteFunc: w.buildEndpointSlicesEventHandlerForDelete(), + UpdateFunc: w.buildEndpointSlicesEventHandlerForUpdate(), + } + + _, err = servicesInformer.AddEventHandler(serviceHandlers) + if err != nil { + return fmt.Errorf(`error occurred adding service event handlers: %w`, err) + } + + _, err = w.endpointSliceInformer.AddEventHandler(endpointSliceHandlers) if err != nil { - return fmt.Errorf(`error occurred adding event handlers: %w`, err) + return fmt.Errorf(`error occurred adding endpoint slice event handlers: %w`, err) } return nil diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 6c78297..b39a989 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -21,5 +21,5 @@ func TestWatcher_ErrWithNilInformer(t *testing.T) { func buildWatcherWithNilInformer() (*Watcher, error) { handler := &mocks.MockHandler{} - return NewWatcher(configuration.Settings{}, handler, nil) + return NewWatcher(configuration.Settings{}, handler, nil, nil) } From 22d2d201a55b611b34cb9a6e429a73c3df3135a5 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 14 Nov 2024 09:44:58 -0700 Subject: [PATCH 088/136] NLB-5872 Added node informer to watcher Any node event from the k8s cluster will cause a service event for every registered service to be added to the handler's rate-limited work queue. --- cmd/nginx-loadbalancer-kubernetes/main.go | 3 +- internal/observation/register.go | 14 ++++++ internal/observation/watcher.go | 60 +++++++++++++++++++++++ internal/observation/watcher_test.go | 4 +- 4 files changed, 78 insertions(+), 3 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 21c609c..21f3e09 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -60,12 +60,13 @@ func run() error { serviceInformer := factory.Core().V1().Services() endpointSliceInformer := factory.Discovery().V1().EndpointSlices() + nodesInformer := factory.Core().V1().Nodes() handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) - watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer) + watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer, nodesInformer) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } diff --git a/internal/observation/register.go b/internal/observation/register.go index 26ab86d..bfe61f8 100644 --- a/internal/observation/register.go +++ b/internal/observation/register.go @@ -47,3 +47,17 @@ func (r *register) getService(namespace string, serviceName string) (*v1.Service s, ok := r.services[registerKey{namespace: namespace, serviceName: serviceName}] return s, ok } + +// listServices returns all the services in the register +func (r *register) listServices() []*v1.Service { + r.mu.RLock() + defer r.mu.RUnlock() + + services := make([]*v1.Service, 0, len(r.services)) + + for _, service := range r.services { + services = append(services, service) + } + + return services +} diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 9752b38..21d6e02 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -36,6 +36,9 @@ type Watcher struct { // endpointSliceInformer is the informer used to watch for changes to endpoint slices endpointSliceInformer cache.SharedIndexInformer + // nodesInformer is the informer used to watch for changes to nodes + nodesInformer cache.SharedIndexInformer + register *register } @@ -45,6 +48,7 @@ func NewWatcher( handler HandlerInterface, serviceInformer coreinformers.ServiceInformer, endpointSliceInformer discoveryinformers.EndpointSliceInformer, + nodeInformer coreinformers.NodeInformer, ) (*Watcher, error) { if serviceInformer == nil { return nil, fmt.Errorf("service informer cannot be nil") @@ -54,14 +58,20 @@ func NewWatcher( return nil, fmt.Errorf("endpoint slice informer cannot be nil") } + if nodeInformer == nil { + return nil, fmt.Errorf("node informer cannot be nil") + } + servicesInformer := serviceInformer.Informer() endpointSlicesInformer := endpointSliceInformer.Informer() + nodesInformer := nodeInformer.Informer() w := &Watcher{ handler: handler, settings: settings, servicesInformer: servicesInformer, endpointSliceInformer: endpointSlicesInformer, + nodesInformer: nodesInformer, register: newRegister(), } @@ -98,9 +108,46 @@ func (w *Watcher) isDesiredService(service *v1.Service) bool { return annotation == w.settings.Watcher.ServiceAnnotation } +func (w *Watcher) buildNodesEventHandlerForAdd() func(interface{}) { + slog.Info("Watcher::buildNodesEventHandlerForAdd") + return func(obj interface{}) { + slog.Debug("received node add event") + for _, service := range w.register.listServices() { + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } + } +} + +func (w *Watcher) buildNodesEventHandlerForUpdate() func(interface{}, interface{}) { + slog.Info("Watcher::buildNodesEventHandlerForUpdate") + return func(previous, updated interface{}) { + slog.Debug("received node update event") + for _, service := range w.register.listServices() { + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } + } +} + +func (w *Watcher) buildNodesEventHandlerForDelete() func(interface{}) { + slog.Info("Watcher::buildNodesEventHandlerForDelete") + return func(obj interface{}) { + slog.Debug("received node delete event") + for _, service := range w.register.listServices() { + var previousService *v1.Service + e := core.NewEvent(core.Updated, service, previousService) + w.handler.AddRateLimitedEvent(&e) + } + } +} + func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { slog.Info("Watcher::buildEndpointSlicesEventHandlerForAdd") return func(obj interface{}) { + slog.Debug("received endpoint slice add event") endpointSlice, ok := obj.(*discovery.EndpointSlice) if !ok { slog.Error("could not convert event object to EndpointSlice", "obj", obj) @@ -122,6 +169,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, interface{}) { slog.Info("Watcher::buildEndpointSlicesEventHandlerForUpdate") return func(previous, updated interface{}) { + slog.Debug("received endpoint slice update event") endpointSlice, ok := updated.(*discovery.EndpointSlice) if !ok { slog.Error("could not convert event object to EndpointSlice", "obj", updated) @@ -143,6 +191,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, i func (w *Watcher) buildEndpointSlicesEventHandlerForDelete() func(interface{}) { slog.Info("Watcher::buildEndpointSlicesEventHandlerForDelete") return func(obj interface{}) { + slog.Debug("received endpoint slice delete event") endpointSlice, ok := obj.(*discovery.EndpointSlice) if !ok { slog.Error("could not convert event object to EndpointSlice", "obj", obj) @@ -235,6 +284,12 @@ func (w *Watcher) initializeEventListeners( UpdateFunc: w.buildEndpointSlicesEventHandlerForUpdate(), } + nodeHandlers := cache.ResourceEventHandlerFuncs{ + AddFunc: w.buildNodesEventHandlerForAdd(), + DeleteFunc: w.buildNodesEventHandlerForDelete(), + UpdateFunc: w.buildNodesEventHandlerForUpdate(), + } + _, err = servicesInformer.AddEventHandler(serviceHandlers) if err != nil { return fmt.Errorf(`error occurred adding service event handlers: %w`, err) @@ -245,5 +300,10 @@ func (w *Watcher) initializeEventListeners( return fmt.Errorf(`error occurred adding endpoint slice event handlers: %w`, err) } + _, err = w.nodesInformer.AddEventHandler(nodeHandlers) + if err != nil { + return fmt.Errorf(`error occurred adding node event handlers: %w`, err) + } + return nil } diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index b39a989..4618408 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestWatcher_ErrWithNilInformer(t *testing.T) { +func TestWatcher_ErrWithNilInformers(t *testing.T) { t.Parallel() _, err := buildWatcherWithNilInformer() require.Error(t, err, "expected construction of watcher with nil informer to fail") @@ -21,5 +21,5 @@ func TestWatcher_ErrWithNilInformer(t *testing.T) { func buildWatcherWithNilInformer() (*Watcher, error) { handler := &mocks.MockHandler{} - return NewWatcher(configuration.Settings{}, handler, nil, nil) + return NewWatcher(configuration.Settings{}, handler, nil, nil, nil) } From 4eb897c3a32042819c296ed491c2a454ba1a3123 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 15 Nov 2024 10:03:12 -0700 Subject: [PATCH 089/136] NLB-5872 Translator now uses shared listers to access endpoint slices and node ports This will save us from redundant kubernetes API calls. --- cmd/nginx-loadbalancer-kubernetes/main.go | 5 +- go.mod | 2 +- internal/observation/handler.go | 12 +- internal/observation/handler_test.go | 5 +- internal/translation/translator.go | 61 +++-- internal/translation/translator_test.go | 279 +++++++++++++--------- 6 files changed, 212 insertions(+), 152 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 21f3e09..b7381f0 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -60,11 +60,14 @@ func run() error { serviceInformer := factory.Core().V1().Services() endpointSliceInformer := factory.Discovery().V1().EndpointSlices() + endpointSliceLister := endpointSliceInformer.Lister() nodesInformer := factory.Core().V1().Nodes() + nodesLister := nodesInformer.Lister() handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translation.NewTranslator(k8sClient)) + translator := translation.NewTranslator(endpointSliceLister, nodesLister) + handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translator) watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer, nodesInformer) if err != nil { diff --git a/go.mod b/go.mod index 6b26183..94a1319 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - golang.org/x/net v0.23.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 @@ -55,6 +54,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect diff --git a/internal/observation/handler.go b/internal/observation/handler.go index 2b5bcfb..45ae6c4 100644 --- a/internal/observation/handler.go +++ b/internal/observation/handler.go @@ -47,7 +47,7 @@ type Handler struct { } type Translator interface { - Translate(context.Context, *core.Event) (core.ServerUpdateEvents, error) + Translate(*core.Event) (core.ServerUpdateEvents, error) } // NewHandler creates a new event handler @@ -76,7 +76,7 @@ func (h *Handler) Run(ctx context.Context) { slog.Debug("Handler::Run") worker := func() { - for h.handleNextEvent(ctx) { + for h.handleNextEvent() { // TODO: Add Telemetry } } @@ -95,11 +95,11 @@ func (h *Handler) ShutDown() { } // handleEvent feeds translated events to the synchronizer -func (h *Handler) handleEvent(ctx context.Context, e *core.Event) error { +func (h *Handler) handleEvent(e *core.Event) error { slog.Debug("Handler::handleEvent", "event", e) // TODO: Add Telemetry - events, err := h.translator.Translate(ctx, e) + events, err := h.translator.Translate(e) if err != nil { return fmt.Errorf(`Handler::handleEvent error translating: %v`, err) } @@ -110,7 +110,7 @@ func (h *Handler) handleEvent(ctx context.Context, e *core.Event) error { } // handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (h *Handler) handleNextEvent(ctx context.Context) bool { +func (h *Handler) handleNextEvent() bool { evt, quit := h.eventQueue.Get() slog.Debug("Handler::handleNextEvent", "event", evt, "quit", quit) if quit { @@ -120,7 +120,7 @@ func (h *Handler) handleNextEvent(ctx context.Context) bool { defer h.eventQueue.Done(evt) event := evt.(*core.Event) - h.withRetry(h.handleEvent(ctx, event), event) + h.withRetry(h.handleEvent(event), event) return true } diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go index 9dd736c..550b318 100644 --- a/internal/observation/handler_test.go +++ b/internal/observation/handler_test.go @@ -6,7 +6,6 @@ package observation import ( - "context" "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" @@ -34,7 +33,7 @@ func TestHandler_AddsEventToSynchronizer(t *testing.T) { handler.AddRateLimitedEvent(event) - handler.handleNextEvent(context.Background()) + handler.handleNextEvent() if len(synchronizer.Events) != 1 { t.Errorf(`handler.AddRateLimitedEvent did not add the event to the queue`) @@ -54,6 +53,6 @@ func buildHandler() ( type fakeTranslator struct{} -func (t *fakeTranslator) Translate(ctx context.Context, event *core.Event) (core.ServerUpdateEvents, error) { +func (t *fakeTranslator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { return core.ServerUpdateEvents{{}}, nil } diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 6d7928d..117c0b9 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -6,7 +6,6 @@ package translation import ( - "context" "fmt" "log/slog" "strings" @@ -14,24 +13,32 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" ) type Translator struct { - k8sClient kubernetes.Interface + endpointSliceLister discoverylisters.EndpointSliceLister + nodeLister corelisters.NodeLister } -func NewTranslator(k8sClient kubernetes.Interface) *Translator { - return &Translator{k8sClient} +func NewTranslator( + endpointSliceLister discoverylisters.EndpointSliceLister, + nodeLister corelisters.NodeLister, +) *Translator { + return &Translator{ + endpointSliceLister: endpointSliceLister, + nodeLister: nodeLister, + } } // Translate transforms event data into an intermediate format that can be consumed by the BorderClient implementations // and used to update the Border Servers. -func (t *Translator) Translate(ctx context.Context, event *core.Event) (core.ServerUpdateEvents, error) { +func (t *Translator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { slog.Debug("Translate::Translate") - return t.buildServerUpdateEvents(ctx, event.Service.Spec.Ports, event) + return t.buildServerUpdateEvents(event.Service.Spec.Ports, event) } // buildServerUpdateEvents builds a list of ServerUpdateEvents based on the event type @@ -40,15 +47,15 @@ func (t *Translator) Translate(ctx context.Context, event *core.Event) (core.Ser // and the list of servers in NGINX+. // The NGINX+ Client uses a single server for Deleted events; // so the list of servers is broken up into individual events. -func (t *Translator) buildServerUpdateEvents(ctx context.Context, ports []v1.ServicePort, event *core.Event, +func (t *Translator) buildServerUpdateEvents(ports []v1.ServicePort, event *core.Event, ) (events core.ServerUpdateEvents, err error) { slog.Debug("Translate::buildServerUpdateEvents", "ports", ports) switch event.Service.Spec.Type { case v1.ServiceTypeNodePort: - return t.buildNodeIPEvents(ctx, ports, event) + return t.buildNodeIPEvents(ports, event) case v1.ServiceTypeClusterIP: - return t.buildClusterIPEvents(ctx, event) + return t.buildClusterIPEvents(event) default: return events, fmt.Errorf("unsupported service type: %s", event.Service.Spec.Type) } @@ -59,8 +66,7 @@ type upstream struct { name string } -func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event, -) (events core.ServerUpdateEvents, err error) { +func (t *Translator) buildClusterIPEvents(event *core.Event) (events core.ServerUpdateEvents, err error) { namespace := event.Service.GetObjectMeta().GetNamespace() serviceName := event.Service.Name @@ -79,8 +85,14 @@ func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event return events, nil } - s := t.k8sClient.DiscoveryV1().EndpointSlices(namespace) - list, err := s.List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("kubernetes.io/service-name=%s", serviceName)}) + lister := t.endpointSliceLister.EndpointSlices(namespace) + selector, err := labels.Parse(fmt.Sprintf("kubernetes.io/service-name=%s", serviceName)) + if err != nil { + logger.Error(`error occurred parsing the selector`, "error", err) + return events, err + } + + list, err := lister.List(selector) if err != nil { logger.Error(`error occurred retrieving the list of endpoint slices`, "error", err) return events, err @@ -88,7 +100,7 @@ func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event upstreams := make(map[upstream][]*core.UpstreamServer) - for _, endpointSlice := range list.Items { + for _, endpointSlice := range list { for _, port := range endpointSlice.Ports { if port.Name == nil || port.Port == nil { continue @@ -124,7 +136,7 @@ func (t *Translator) buildClusterIPEvents(ctx context.Context, event *core.Event return events, nil } -func (t *Translator) buildNodeIPEvents(ctx context.Context, ports []v1.ServicePort, event *core.Event, +func (t *Translator) buildNodeIPEvents(ports []v1.ServicePort, event *core.Event, ) (core.ServerUpdateEvents, error) { slog.Debug("Translate::buildNodeIPEvents", "ports", ports) @@ -136,7 +148,7 @@ func (t *Translator) buildNodeIPEvents(ctx context.Context, ports []v1.ServicePo continue } - addresses, err := t.retrieveNodeIps(ctx) + addresses, err := t.retrieveNodeIps() if err != nil { return nil, err } @@ -192,21 +204,26 @@ func getContextAndUpstreamName(portName string) (clientType string, appName stri // notMasterNode retrieves the IP Addresses of the nodes in the cluster. Currently, the master node is excluded. This is // because the master node may or may not be a worker node and thus may not be able to route traffic. -func (t *Translator) retrieveNodeIps(ctx context.Context) ([]string, error) { +func (t *Translator) retrieveNodeIps() ([]string, error) { started := time.Now() slog.Debug("Translator::retrieveNodeIps") var nodeIps []string - nodes, err := t.k8sClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + nodes, err := t.nodeLister.List(labels.Everything()) if err != nil { slog.Error("error occurred retrieving the list of nodes", "error", err) return nil, err } - for _, node := range nodes.Items { + for _, node := range nodes { + if node == nil { + slog.Error("list contains nil node") + continue + } + // this is kind of a broad assumption, should probably make this a configurable option - if notMasterNode(node) { + if notMasterNode(*node) { for _, address := range node.Status.Addresses { if address.Type == v1.NodeInternalIP { nodeIps = append(nodeIps, address.Address) diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index 73c99a4..c6d42f5 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -13,11 +13,12 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/pointer" - "golang.org/x/net/context" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" ) const ( @@ -52,9 +53,12 @@ func TestCreatedTranslateNoPorts(t *testing.T) { service := defaultService(tc.serviceType) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient([]discovery.EndpointSlice{}, []v1.Node{})) + translator := NewTranslator( + NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), + NewFakeNodeLister([]*v1.Node{}, nil), + ) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -86,9 +90,12 @@ func TestCreatedTranslateNoInterestingPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient([]discovery.EndpointSlice{}, []v1.Node{})) + translator := NewTranslator( + NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), + NewFakeNodeLister([]*v1.Node{}, nil), + ) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -106,8 +113,8 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { t.Parallel() testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -134,8 +141,8 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -155,8 +162,8 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { t.Parallel() testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -183,8 +190,8 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -205,8 +212,8 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -234,8 +241,8 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -255,8 +262,8 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -283,8 +290,8 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildCreatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -308,8 +315,8 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -333,8 +340,8 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { service := defaultService(tc.serviceType) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -352,8 +359,8 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -379,8 +386,8 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -398,8 +405,8 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -425,8 +432,8 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -447,8 +454,8 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -474,8 +481,8 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -496,8 +503,8 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -524,8 +531,8 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -546,8 +553,8 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice expectedServerCount int }{ "nodePort": { @@ -574,8 +581,8 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildUpdatedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -599,8 +606,8 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -620,8 +627,8 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { service := defaultService(tc.serviceType) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -641,8 +648,8 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -663,8 +670,8 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -685,8 +692,8 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -709,8 +716,8 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -731,8 +738,8 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -754,8 +761,8 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -775,8 +782,8 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -800,8 +807,8 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -822,8 +829,8 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -845,8 +852,8 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { service := defaultService(tc.serviceType) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -866,8 +873,8 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -890,8 +897,8 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -912,8 +919,8 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -937,8 +944,8 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -959,8 +966,8 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -984,8 +991,8 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1005,8 +1012,8 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1030,8 +1037,8 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1052,8 +1059,8 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1074,8 +1081,8 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { service := defaultService(tc.serviceType) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1095,8 +1102,8 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1120,8 +1127,8 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1142,8 +1149,8 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1166,8 +1173,8 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1188,8 +1195,8 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1212,8 +1219,8 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1233,8 +1240,8 @@ func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType - nodes []v1.Node - endpoints []discovery.EndpointSlice + nodes []*v1.Node + endpoints []*discovery.EndpointSlice }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1258,8 +1265,8 @@ func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { service := serviceWithPorts(tc.serviceType, ports) event := buildDeletedEvent(service) - translator := NewTranslator(NewFakeClient(tc.endpoints, tc.nodes)) - translatedEvents, err := translator.Translate(context.TODO(), &event) + translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) + translatedEvents, err := translator.Translate(&event) if err != nil { t.Fatalf(TranslateErrorFormat, err) } @@ -1324,9 +1331,9 @@ func buildEvent(eventType core.EventType, service *v1.Service) core.Event { return event } -func generateNodes(count int) (nodes []v1.Node) { +func generateNodes(count int) (nodes []*v1.Node) { for i := 0; i < count; i++ { - nodes = append(nodes, v1.Node{ + nodes = append(nodes, &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("node%d", i), }, @@ -1345,7 +1352,7 @@ func generateNodes(count int) (nodes []v1.Node) { } func generateEndpointSlices(endpointCount, portCount, updatablePortCount int, -) (endpointSlices []discovery.EndpointSlice) { +) (endpointSlices []*discovery.EndpointSlice) { servicePorts := generateUpdatablePorts(portCount, updatablePortCount) ports := make([]discovery.EndpointPort, 0, len(servicePorts)) @@ -1365,7 +1372,7 @@ func generateEndpointSlices(endpointCount, portCount, updatablePortCount int, }) } - endpointSlices = append(endpointSlices, discovery.EndpointSlice{ + endpointSlices = append(endpointSlices, &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: "endpointSlice", Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, @@ -1415,13 +1422,47 @@ func generateUpdatablePorts(portCount int, updatableCount int) []v1.ServicePort return ports } -func NewFakeClient(endpointSlices []discovery.EndpointSlice, nodes []v1.Node) *fake.Clientset { - return fake.NewSimpleClientset( - &discovery.EndpointSliceList{ - Items: endpointSlices, - }, - &v1.NodeList{ - Items: nodes, - }, - ) +func NewFakeEndpointSliceLister(list []*discovery.EndpointSlice, err error) discoverylisters.EndpointSliceLister { + return &endpointSliceLister{ + list: list, + err: err, + } +} + +func NewFakeNodeLister(list []*v1.Node, err error) corelisters.NodeLister { + return &nodeLister{ + list: list, + err: err, + } +} + +type nodeLister struct { + list []*v1.Node + err error +} + +func (l *nodeLister) List(selector labels.Selector) (ret []*v1.Node, err error) { + return l.list, l.err +} + +// currently unused +func (l *nodeLister) Get(name string) (*v1.Node, error) { + return nil, nil +} + +type endpointSliceLister struct { + list []*discovery.EndpointSlice + err error +} + +func (l *endpointSliceLister) List(selector labels.Selector) (ret []*discovery.EndpointSlice, err error) { + return l.list, l.err +} + +func (l *endpointSliceLister) Get(name string) (*discovery.EndpointSlice, error) { + return nil, nil +} + +func (l *endpointSliceLister) EndpointSlices(name string) discoverylisters.EndpointSliceNamespaceLister { + return l } From 9775550cdaf0fb1113dfdfb0a5da9f25ac7b5d2a Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 19 Nov 2024 15:04:45 -0700 Subject: [PATCH 090/136] NLB-5872 If user removes nginxaas service annotation remove the service from the watcher's registe nodePort: 31575r --- internal/observation/watcher.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 21d6e02..83fc37a 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -252,14 +252,20 @@ func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interfac slog.Info("Watcher::buildServiceEventHandlerForUpdate") return func(previous, updated interface{}) { // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API + previousService := previous.(*v1.Service) service := updated.(*v1.Service) + + if w.isDesiredService(previousService) && !w.isDesiredService(service) { + w.register.removeService(previousService) + return + } + if !w.isDesiredService(service) { return } w.register.addOrUpdateService(service) - previousService := previous.(*v1.Service) e := core.NewEvent(core.Updated, service, previousService) w.handler.AddRateLimitedEvent(&e) } From b962c8108a4024e41b991acb4fa8fac60c74ba4b Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 15 Nov 2024 10:04:26 -0700 Subject: [PATCH 091/136] NLB-5872 Bumped version to 0.8.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 39e898a..a3df0a6 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.7.1 +0.8.0 From da73e2700c969977f6c17f2367b3eddc6d2365a4 Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 8 Oct 2024 15:42:04 -0700 Subject: [PATCH 092/136] Cleanup Whitesource scan We need to set the product prefix needs to be set so we can view Whitesource scans properly for a given Whitesource project. Also, cleaned up some vars that are not needed here as we have defaults in the generic template. --- .gitlab-ci.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8237868..21fa6b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -147,14 +147,8 @@ whitesource-scan: # - .go-cache - .whitesource-template-go variables: - NESTED: "true" - WS_WSS_URL: "https://f5.whitesourcesoftware.com/agent" - WS_APIKEY: "${WS_APIKEY_NGINX}" - WS_PRODUCTNAME: "N4A" - WS_PROJECTNAME: "${CI_PROJECT_NAME}" - WS_GO_MODULES_RESOLVEDEPENDENCIES: "true" - WS_GO_RESOLVEDEPENDENCIES: "false" - WS_GENERATEPROJECTDETAILSJSON: "true" + PRODUCT_PREFIX: "n4a" + WS_PROJECT: "${CI_PROJECT_NAME}" script: - *golang-private - !reference [.whitesource-template-go, script] From 1e88bdee8811a3a85fa9304a30af42068a9b07a8 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 8 Aug 2024 10:22:53 -0700 Subject: [PATCH 093/136] NLB-5384: Redeploy operator to run tests --- .gitlab-ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21fa6b6..375a262 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,7 @@ include: stages: - lint+test+build - security scanning + - e2e-tests - release - release-cnab @@ -218,6 +219,22 @@ container_scanning: when: never - when: on_success +trigger-e2e: + stage: e2e-tests + variables: + IS_TEST_ONLY: "true" + TEST_TYPE: "e2e.arm" + TEST_NLK_CHART_URL: "oci://${DEVOPS_DOCKER_URL_DEFAULT}/nginx-azure-lb/${CI_PROJECT_NAME}/charts/${CI_COMMIT_REF_SLUG}/nginxaas-loadbalancer-kubernetes" + TEST_NLK_IMG_TAG: ${CI_COMMIT_SHORT_SHA} + TEST_ARGS: "test_nlk.py" + AZ_LOCATION: "West Central US" + trigger: + project: f5/nginx/nginxazurelb/tools/nlbtest + branch: main + strategy: depend + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + .release-common: stage: release image: $DEVTOOLS_IMG From 5f0982f53ff7904c0366bbbd09be448b7315f69e Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 6 Dec 2024 16:01:27 -0800 Subject: [PATCH 094/136] Update chart,app version to 0.8.0 --- charts/nlk/Chart.yaml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/charts/nlk/Chart.yaml b/charts/nlk/Chart.yaml index 4e1860f..b11ab86 100644 --- a/charts/nlk/Chart.yaml +++ b/charts/nlk/Chart.yaml @@ -1,17 +1,16 @@ --- apiVersion: v2 -appVersion: 0.4.0 +appVersion: 0.8.0 description: NGINXaaS LoadBalancer for Kubernetes name: nginxaas-loadbalancer-kubernetes keywords: -- nginx -- nginxaas -- loadbalancer + - nginx + - nginxaas + - loadbalancer kubeVersion: '>= 1.22.0-0' maintainers: -- name: "@ciroque" -- name: "@chrisakker" -- name: "@abdennour" - + - name: "@ciroque" + - name: "@chrisakker" + - name: "@abdennour" type: application -version: 0.4.0 +version: 0.8.0 From ef8ac000f5a94e1553005579c4e77e1d82b59f68 Mon Sep 17 00:00:00 2001 From: sarna Date: Fri, 6 Dec 2024 16:08:36 -0800 Subject: [PATCH 095/136] Fix chart,app version for AKS marketplace While nothing breaks if the versions are not aligned inside the AKS marketplace application bundle, the side effect is that when a user review the helm releases on an AKS cluster, they see the hardcoded app and chart version which is misleading. This commit fixes the chart and app version inside the Helm chart so that the user sees correct information while interacting with their AKS cluster. --- scripts/cnab.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/cnab.sh b/scripts/cnab.sh index 7d3c0b0..da6bdb4 100755 --- a/scripts/cnab.sh +++ b/scripts/cnab.sh @@ -25,6 +25,7 @@ set_version() { update_helm_chart() { yq -ie '.global.azure.images.nlk.registry = .nlk.image.registry | .global.azure.images.nlk.image = .nlk.image.repository | .global.azure.images.nlk.tag = env(VERSION)' charts/nlk/values.yaml + yq -ie '.version = env(VERSION) | .appVersion = env(VERSION)' charts/nlk/Chart.yaml } update_bundle() { From 9d5c093f90faf694f505ef1731952111ab6c3bc5 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 9 Dec 2024 14:47:51 -0800 Subject: [PATCH 096/136] Update default logger to be info The default log-level of "warn" does not help much even when there are issues with nlk: - While talking to the API. - updating upstreams. This commit updates the default loglevel to be info so that the customer sees useful logs while debugging the component. The user can choose to turn it down if they want to but having a slightly louder log level helps with the initial experience especially when the product is new (ish). --- charts/nlk/values.yaml | 2 +- version | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/nlk/values.yaml b/charts/nlk/values.yaml index 851f67d..1f2f1f2 100644 --- a/charts/nlk/values.yaml +++ b/charts/nlk/values.yaml @@ -31,7 +31,7 @@ nlk: annotations: {} config: ## trace,debug,info,warn,error,fatal,panic - # logLevel: "warn" + logLevel: "info" ## the nginx hosts (comma-separated) to send upstream updates to nginxHosts: "" diff --git a/version b/version index a3df0a6..6f4eebd 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.0 +0.8.1 From 285af21583cc7bebf865d38654eaa1798d200e87 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 11 Dec 2024 08:48:04 -0700 Subject: [PATCH 097/136] NLB-5933 Remove handler; responsibilities for looking up upstream servers moved to synchronizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The synchronizer now assumes the responsibility of calling the translator to find the server addresses that need to be updated and handling errors by re-adding the work item to the queue. It continues to handle errors from the border client by requeuing workitems. The synchronizer needs to use a service identifier as the key in the workqueue (service ID=service name+namespace). If a workitem is dequeued, the synchronizer looks up the service in the shared informer cache to gain the most up-to-date state. The only time we use our local cache to discover the last known state of the service is in the case of a service deletion. The shared informer cache will no longer contain data on the deleted service, so we need to rely on our own local event cache to find details of the service's ports to determine the upstream and context, etc. Any incoming events overwrite stale events in the local event cache. Because the synchronizer always refers to the shared informer’s cache to find the current service addresses when the work item is dequeued we reduce the risk of applying stale state. This change simplifies event handling: there is only one rate-limited workqueue to be configured and managed, and we don't have to worry about potentially stale state being dequeued from two separate queues. --- cmd/nginx-loadbalancer-kubernetes/main.go | 17 +- internal/observation/handler.go | 141 -------------- internal/observation/handler_test.go | 58 ------ internal/observation/watcher.go | 28 +-- internal/observation/watcher_test.go | 4 +- internal/synchronization/cache.go | 40 ++++ internal/synchronization/synchronizer.go | 179 +++++++++++------- internal/synchronization/synchronizer_test.go | 156 ++++++++++++--- test/mocks/mock_handler.go | 26 --- 9 files changed, 304 insertions(+), 345 deletions(-) delete mode 100644 internal/observation/handler.go delete mode 100644 internal/observation/handler_test.go create mode 100644 internal/synchronization/cache.go delete mode 100644 test/mocks/mock_handler.go diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index b7381f0..501d7c4 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -49,11 +49,6 @@ func run() error { synchronizerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - synchronizer, err := synchronization.NewSynchronizer(settings, synchronizerWorkqueue) - if err != nil { - return fmt.Errorf(`error initializing synchronizer: %w`, err) - } - factory := informers.NewSharedInformerFactoryWithOptions( k8sClient, settings.Watcher.ResyncPeriod, ) @@ -64,12 +59,15 @@ func run() error { nodesInformer := factory.Core().V1().Nodes() nodesLister := nodesInformer.Lister() - handlerWorkqueue := buildWorkQueue(settings.Synchronizer.WorkQueueSettings) - translator := translation.NewTranslator(endpointSliceLister, nodesLister) - handler := observation.NewHandler(settings, synchronizer, handlerWorkqueue, translator) - watcher, err := observation.NewWatcher(settings, handler, serviceInformer, endpointSliceInformer, nodesInformer) + synchronizer, err := synchronization.NewSynchronizer( + settings, synchronizerWorkqueue, translator, serviceInformer.Lister()) + if err != nil { + return fmt.Errorf(`error initializing synchronizer: %w`, err) + } + + watcher, err := observation.NewWatcher(settings, synchronizer, serviceInformer, endpointSliceInformer, nodesInformer) if err != nil { return fmt.Errorf(`error occurred creating a watcher: %w`, err) } @@ -82,7 +80,6 @@ func run() error { } } - go handler.Run(ctx) go synchronizer.Run(ctx.Done()) probeServer := probation.NewHealthServer() diff --git a/internal/observation/handler.go b/internal/observation/handler.go deleted file mode 100644 index 45ae6c4..0000000 --- a/internal/observation/handler.go +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package observation - -import ( - "context" - "fmt" - "log/slog" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/workqueue" -) - -// HandlerInterface is the interface for the event handler -type HandlerInterface interface { - // AddRateLimitedEvent defines the interface for adding an event to the event queue - AddRateLimitedEvent(event *core.Event) - - // Run defines the interface used to start the event handler - Run(ctx context.Context) - - // ShutDown defines the interface used to stop the event handler - ShutDown() -} - -// Handler is responsible for processing events in the "nlk-handler" queue. -// When processing a message the Translation module is used to translate the event into an internal representation. -// The translation process may result in multiple events being generated. This fan-out mainly supports the differences -// in NGINX Plus API calls for creating/updating Upstreams and deleting Upstreams. -type Handler struct { - // eventQueue is the queue used to store events - eventQueue workqueue.RateLimitingInterface - - // settings is the configuration settings - settings configuration.Settings - - // synchronizer is the synchronizer used to synchronize the internal representation with a Border Server - synchronizer synchronization.Interface - - translator Translator -} - -type Translator interface { - Translate(*core.Event) (core.ServerUpdateEvents, error) -} - -// NewHandler creates a new event handler -func NewHandler( - settings configuration.Settings, - synchronizer synchronization.Interface, - eventQueue workqueue.RateLimitingInterface, - translator Translator, -) *Handler { - return &Handler{ - eventQueue: eventQueue, - settings: settings, - synchronizer: synchronizer, - translator: translator, - } -} - -// AddRateLimitedEvent adds an event to the event queue -func (h *Handler) AddRateLimitedEvent(event *core.Event) { - slog.Debug(`Handler::AddRateLimitedEvent`, "event", event) - h.eventQueue.AddRateLimited(event) -} - -// Run starts the event handler, spins up Goroutines to process events, and waits for context to be done -func (h *Handler) Run(ctx context.Context) { - slog.Debug("Handler::Run") - - worker := func() { - for h.handleNextEvent() { - // TODO: Add Telemetry - } - } - - for i := 0; i < h.settings.Handler.Threads; i++ { - go wait.Until(worker, 0, ctx.Done()) - } - - <-ctx.Done() -} - -// ShutDown stops the event handler and shuts down the event queue -func (h *Handler) ShutDown() { - slog.Debug("Handler::ShutDown") - h.eventQueue.ShutDown() -} - -// handleEvent feeds translated events to the synchronizer -func (h *Handler) handleEvent(e *core.Event) error { - slog.Debug("Handler::handleEvent", "event", e) - // TODO: Add Telemetry - - events, err := h.translator.Translate(e) - if err != nil { - return fmt.Errorf(`Handler::handleEvent error translating: %v`, err) - } - - h.synchronizer.AddEvents(events) - - return nil -} - -// handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (h *Handler) handleNextEvent() bool { - evt, quit := h.eventQueue.Get() - slog.Debug("Handler::handleNextEvent", "event", evt, "quit", quit) - if quit { - return false - } - - defer h.eventQueue.Done(evt) - - event := evt.(*core.Event) - h.withRetry(h.handleEvent(event), event) - - return true -} - -// withRetry handles errors from the event handler and requeues events that fail -func (h *Handler) withRetry(err error, event *core.Event) { - slog.Debug("Handler::withRetry") - if err != nil { - // TODO: Add Telemetry - if h.eventQueue.NumRequeues(event) < h.settings.Handler.RetryCount { - h.eventQueue.AddRateLimited(event) - slog.Info("Handler::withRetry: requeued event", "event", event, "error", err) - } else { - h.eventQueue.Forget(event) - slog.Warn(`Handler::withRetry: event has been dropped due to too many retries`, "event", event) - } - } // TODO: Add error logging -} diff --git a/internal/observation/handler_test.go b/internal/observation/handler_test.go deleted file mode 100644 index 550b318..0000000 --- a/internal/observation/handler_test.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package observation - -import ( - "testing" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" - v1 "k8s.io/api/core/v1" -) - -func TestHandler_AddsEventToSynchronizer(t *testing.T) { - t.Parallel() - synchronizer, handler := buildHandler() - - event := &core.Event{ - Type: core.Created, - Service: &v1.Service{ - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Name: "http-back", - }, - }, - }, - }, - } - - handler.AddRateLimitedEvent(event) - - handler.handleNextEvent() - - if len(synchronizer.Events) != 1 { - t.Errorf(`handler.AddRateLimitedEvent did not add the event to the queue`) - } -} - -func buildHandler() ( - *mocks.MockSynchronizer, *Handler, -) { - eventQueue := &mocks.MockRateLimiter{} - synchronizer := &mocks.MockSynchronizer{} - - handler := NewHandler(configuration.Settings{}, synchronizer, eventQueue, &fakeTranslator{}) - - return synchronizer, handler -} - -type fakeTranslator struct{} - -func (t *fakeTranslator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { - return core.ServerUpdateEvents{{}}, nil -} diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 83fc37a..6c8977e 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -12,6 +12,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" + "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" v1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -24,8 +25,7 @@ import ( // Particularly, Services in the namespace defined in the WatcherSettings::NginxIngressNamespace setting. // When a change is detected, an Event is generated and added to the Handler's queue. type Watcher struct { - // handler is the event handler - handler HandlerInterface + synchronizer synchronization.Interface // settings is the configuration settings settings configuration.Settings @@ -45,7 +45,7 @@ type Watcher struct { // NewWatcher creates a new Watcher func NewWatcher( settings configuration.Settings, - handler HandlerInterface, + synchronizer synchronization.Interface, serviceInformer coreinformers.ServiceInformer, endpointSliceInformer discoveryinformers.EndpointSliceInformer, nodeInformer coreinformers.NodeInformer, @@ -67,7 +67,7 @@ func NewWatcher( nodesInformer := nodeInformer.Informer() w := &Watcher{ - handler: handler, + synchronizer: synchronizer, settings: settings, servicesInformer: servicesInformer, endpointSliceInformer: endpointSlicesInformer, @@ -92,7 +92,7 @@ func (w *Watcher) Run(ctx context.Context) error { slog.Debug("Watcher::Watch") defer utilruntime.HandleCrash() - defer w.handler.ShutDown() + defer w.synchronizer.ShutDown() <-ctx.Done() return nil @@ -115,7 +115,7 @@ func (w *Watcher) buildNodesEventHandlerForAdd() func(interface{}) { for _, service := range w.register.listServices() { var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } } @@ -127,7 +127,7 @@ func (w *Watcher) buildNodesEventHandlerForUpdate() func(interface{}, interface{ for _, service := range w.register.listServices() { var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } } @@ -139,7 +139,7 @@ func (w *Watcher) buildNodesEventHandlerForDelete() func(interface{}) { for _, service := range w.register.listServices() { var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } } @@ -162,7 +162,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -184,7 +184,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, i var previousService *v1.Service e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -206,7 +206,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForDelete() func(interface{}) { var previousService *v1.Service e := core.NewEvent(core.Deleted, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -224,7 +224,7 @@ func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { var previousService *v1.Service e := core.NewEvent(core.Created, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -242,7 +242,7 @@ func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { var previousService *v1.Service e := core.NewEvent(core.Deleted, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } @@ -267,7 +267,7 @@ func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interfac w.register.addOrUpdateService(service) e := core.NewEvent(core.Updated, service, previousService) - w.handler.AddRateLimitedEvent(&e) + w.synchronizer.AddEvent(e) } } diff --git a/internal/observation/watcher_test.go b/internal/observation/watcher_test.go index 4618408..b8e8369 100644 --- a/internal/observation/watcher_test.go +++ b/internal/observation/watcher_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" "github.com/stretchr/testify/require" ) @@ -20,6 +19,5 @@ func TestWatcher_ErrWithNilInformers(t *testing.T) { } func buildWatcherWithNilInformer() (*Watcher, error) { - handler := &mocks.MockHandler{} - return NewWatcher(configuration.Settings{}, handler, nil, nil, nil) + return NewWatcher(configuration.Settings{}, nil, nil, nil, nil) } diff --git a/internal/synchronization/cache.go b/internal/synchronization/cache.go new file mode 100644 index 0000000..6c3f164 --- /dev/null +++ b/internal/synchronization/cache.go @@ -0,0 +1,40 @@ +package synchronization + +import ( + "sync" + + v1 "k8s.io/api/core/v1" +) + +// cache contains the most recent definitions for services monitored by NLK. +// We need these so that if a service is deleted from the shared informer cache, the +// caller can access the spec of the deleted service for cleanup. +type cache struct { + mu sync.RWMutex + store map[ServiceKey]*v1.Service +} + +func newCache() *cache { + return &cache{ + store: make(map[ServiceKey]*v1.Service), + } +} + +func (s *cache) get(key ServiceKey) (*v1.Service, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + service, ok := s.store[key] + return service, ok +} + +func (s *cache) add(key ServiceKey, service *v1.Service) { + s.mu.Lock() + defer s.mu.Unlock() + s.store[key] = service +} + +func (s *cache) delete(key ServiceKey) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.store, key) +} diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 20bc99a..fef9724 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -6,6 +6,7 @@ package synchronization import ( + "errors" "fmt" "log/slog" @@ -14,17 +15,16 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" nginxClient "github.com/nginxinc/nginx-plus-go-client/client" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" + corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/util/workqueue" ) // Interface defines the interface needed to implement a synchronizer. type Interface interface { - // AddEvents adds a list of events to the queue. - AddEvents(events core.ServerUpdateEvents) - // AddEvent adds an event to the queue. - AddEvent(event *core.ServerUpdateEvent) + AddEvent(event core.Event) // Run starts the synchronizer. Run(stopCh <-chan struct{}) @@ -33,60 +33,57 @@ type Interface interface { ShutDown() } +type Translator interface { + Translate(*core.Event) (core.ServerUpdateEvents, error) +} + +type ServiceKey struct { + Name string + Namespace string +} + // Synchronizer is responsible for synchronizing the state of the Border Servers. // Operating against the "nlk-synchronizer", it handles events by creating // a Border Client as specified in the Service annotation for the Upstream. // See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { - eventQueue workqueue.RateLimitingInterface - settings configuration.Settings + eventQueue workqueue.RateLimitingInterface + settings configuration.Settings + translator Translator + cache *cache + serviceLister corelisters.ServiceLister } // NewSynchronizer creates a new Synchronizer. func NewSynchronizer( settings configuration.Settings, eventQueue workqueue.RateLimitingInterface, + translator Translator, + serviceLister corelisters.ServiceLister, ) (*Synchronizer, error) { synchronizer := Synchronizer{ - eventQueue: eventQueue, - settings: settings, + eventQueue: eventQueue, + settings: settings, + cache: newCache(), + translator: translator, + serviceLister: serviceLister, } return &synchronizer, nil } -// AddEvents adds a list of events to the queue. If no hosts are specified this is a null operation. -// Events will fan out to the number of hosts specified before being added to the queue. -func (s *Synchronizer) AddEvents(events core.ServerUpdateEvents) { - slog.Debug(`Synchronizer::AddEvents adding events`, slog.Int("eventCount", len(events))) +// AddEvent adds an event to the rate-limited queue. If no hosts are specified this is a null operation. +func (s *Synchronizer) AddEvent(event core.Event) { + slog.Debug(`Synchronizer::AddEvent`) if len(s.settings.NginxPlusHosts) == 0 { slog.Warn(`No Nginx Plus hosts were specified. Skipping synchronization.`) return } - updatedEvents := s.fanOutEventToHosts(events) - - for _, event := range updatedEvents { - s.AddEvent(event) - } -} - -// AddEvent adds an event to the queue. If no hosts are specified this is a null operation. -// Events will be added to the queue after a random delay between MinMillisecondsJitter and MaxMillisecondsJitter. -func (s *Synchronizer) AddEvent(event *core.ServerUpdateEvent) { - slog.Debug(`Synchronizer::AddEvent`, "event", event) - - if event.NginxHost == `` { - slog.Warn(`Nginx host was not specified. Skipping synchronization.`) - return - } - - after := RandomMilliseconds( - s.settings.Synchronizer.MinMillisecondsJitter, - s.settings.Synchronizer.MaxMillisecondsJitter, - ) - s.eventQueue.AddAfter(event, after) + key := ServiceKey{Name: event.Service.Name, Namespace: event.Service.Namespace} + s.cache.add(key, event.Service) + s.eventQueue.AddRateLimited(key) } // Run starts the Synchronizer, spins up Goroutines to process events, and waits for a stop signal. @@ -129,7 +126,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica // fanOutEventToHosts takes a list of events and returns a list of events, one for each Border Server. func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.ServerUpdateEvents { - slog.Debug(`Synchronizer::fanOutEventToHosts`, "event", event) + slog.Debug(`Synchronizer::fanOutEventToHosts`) var events core.ServerUpdateEvents @@ -145,33 +142,77 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se return events } -// handleEvent dispatches an event to the proper handler function. -func (s *Synchronizer) handleEvent(event *core.ServerUpdateEvent) error { - slog.Debug(`Synchronizer::handleEvent`, slog.String("eventID", event.ID)) - - var err error +// handleServiceEvent gets the latest state for the service from the shared +// informer cache, translates the service event into server update events and +// dispatches these events to the proper handler function. +func (s *Synchronizer) handleServiceEvent(key ServiceKey) (err error) { + logger := slog.With("service", key) + logger.Debug(`Synchronizer::handleServiceEvent`) + + // if a service exists in the shared informer cache, we can assume that we need to update it + event := core.Event{Type: core.Updated} + + namespaceLister := s.serviceLister.Services(key.Namespace) + k8sService, err := namespaceLister.Get(key.Name) + switch { + // the service has been deleted. We need to rely on the local cache to + // gather the last known state of the service so we can delete its + // upstream servers + case err != nil && apierrors.IsNotFound(err): + service, ok := s.cache.get(key) + if !ok { + logger.Warn(`Synchronizer::handleServiceEvent: no information could be gained about service`) + return nil + } + // no matter what type the cached event has, the service no longer exists, so the type is Deleted + event.Type = core.Deleted + event.Service = service + case err != nil: + return err + default: + event.Service = k8sService + } - switch event.Type { - case core.Created: - fallthrough + events, err := s.translator.Translate(&event) + if err != nil { + return err + } - case core.Updated: - err = s.handleCreatedUpdatedEvent(event) + if len(events) == 0 { + slog.Warn("Synchronizer::handleServiceEvent: no events to process") + return nil + } - case core.Deleted: - err = s.handleDeletedEvent(event) + events = s.fanOutEventToHosts(events) + + for _, evt := range events { + switch event.Type { + case core.Created, core.Updated: + if handleErr := s.handleCreatedUpdatedEvent(evt); handleErr != nil { + err = errors.Join(err, handleErr) + } + case core.Deleted: + if handleErr := s.handleDeletedEvent(evt); handleErr != nil { + err = errors.Join(err, handleErr) + } + default: + slog.Warn(`Synchronizer::handleServiceEvent: unknown event type`, "type", event.Type) + } + } - default: - slog.Warn(`Synchronizer::handleEvent: unknown event type`, "type", event.Type) + if err != nil { + return err } - if err == nil { - slog.Info( - "Synchronizer::handleEvent: successfully handled the event", - "type", event.TypeName(), "upstreamName", event.UpstreamName, "eventID", event.ID) + if event.Type == core.Deleted { + s.cache.delete(ServiceKey{Name: event.Service.Name, Namespace: event.Service.Namespace}) } - return err + slog.Debug( + "Synchronizer::handleServiceEvent: successfully handled the service change", "service", key, + ) + + return nil } // handleCreatedUpdatedEvent handles events of type Created or Updated. @@ -210,19 +251,25 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv return nil } -// handleNextEvent pulls an event from the event queue and feeds it to the event handler with retry logic -func (s *Synchronizer) handleNextEvent() bool { - slog.Debug(`Synchronizer::handleNextEvent`) +// handleNextServiceEvent pulls a service from the event queue and feeds it to +// the service event handler with retry logic +func (s *Synchronizer) handleNextServiceEvent() bool { + slog.Debug(`Synchronizer::handleNextServiceEvent`) - evt, quit := s.eventQueue.Get() + svc, quit := s.eventQueue.Get() if quit { return false } - defer s.eventQueue.Done(evt) + defer s.eventQueue.Done(svc) + + key, ok := svc.(ServiceKey) + if !ok { + slog.Warn(`Synchronizer::handleNextServiceEvent: invalid service type`, "service", svc) + return true + } - event := evt.(*core.ServerUpdateEvent) - s.withRetry(s.handleEvent(event), event) + s.withRetry(s.handleServiceEvent(key), key) return true } @@ -230,18 +277,18 @@ func (s *Synchronizer) handleNextEvent() bool { // worker is the main message loop func (s *Synchronizer) worker() { slog.Debug(`Synchronizer::worker`) - for s.handleNextEvent() { + for s.handleNextServiceEvent() { } } // withRetry handles errors from the event handler and requeues events that fail -func (s *Synchronizer) withRetry(err error, event *core.ServerUpdateEvent) { +func (s *Synchronizer) withRetry(err error, key ServiceKey) { slog.Debug("Synchronizer::withRetry") if err != nil { // TODO: Add Telemetry - s.eventQueue.AddRateLimited(event) - slog.Info(`Synchronizer::withRetry: requeued event`, "eventID", event.ID, "error", err) + s.eventQueue.AddRateLimited(key) + slog.Info(`Synchronizer::withRetry: requeued service update`, "service", key, "error", err) } else { - s.eventQueue.Forget(event) + s.eventQueue.Forget(key) } // TODO: Add error logging } diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index d1710b2..7592f5c 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -13,6 +13,10 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" "github.com/nginxinc/kubernetes-nginx-ingress/test/mocks" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" ) func TestSynchronizer_NewSynchronizer(t *testing.T) { @@ -20,7 +24,12 @@ func TestSynchronizer_NewSynchronizer(t *testing.T) { rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(configuration.Settings{}, rateLimiter) + synchronizer, err := NewSynchronizer( + configuration.Settings{}, + rateLimiter, + &fakeTranslator{}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -33,17 +42,15 @@ func TestSynchronizer_NewSynchronizer(t *testing.T) { func TestSynchronizer_AddEventNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 - event := &core.ServerUpdateEvent{ - ID: "", - NginxHost: "", - Type: 0, - UpstreamName: "", - UpstreamServers: nil, - } rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings(), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings(), + rateLimiter, + &fakeTranslator{}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -54,7 +61,7 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { // NOTE: Ideally we have a custom logger that can be mocked to capture the log message // and assert a warning was logged that the NGINX Plus host was not specified. - synchronizer.AddEvent(event) + synchronizer.AddEvent(core.Event{}) actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -64,11 +71,16 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { func TestSynchronizer_AddEventOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 1 - events := buildEvents(1) + events := buildServerUpdateEvents(1) rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings("https://localhost:8080"), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings("https://localhost:8080"), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -77,7 +89,7 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvent(events[0]) + synchronizer.AddEvent(buildServiceUpdateEvent(1)) actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -87,7 +99,7 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { func TestSynchronizer_AddEventManyHosts(t *testing.T) { t.Parallel() const expectedEventCount = 1 - events := buildEvents(1) + events := buildServerUpdateEvents(1) hosts := []string{ "https://localhost:8080", "https://localhost:8081", @@ -96,7 +108,12 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings(hosts...), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings(hosts...), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -105,7 +122,7 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvent(events[0]) + synchronizer.AddEvent(buildServiceUpdateEvent(1)) actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -115,10 +132,15 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { func TestSynchronizer_AddEventsNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 - events := buildEvents(4) + events := buildServerUpdateEvents(4) rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings(), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings(), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -129,7 +151,10 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { // NOTE: Ideally we have a custom logger that can be mocked to capture the log message // and assert a warning was logged that the NGINX Plus host was not specified. - synchronizer.AddEvents(events) + for i := 0; i < 4; i++ { + synchronizer.AddEvent(buildServiceUpdateEvent(i)) + } + actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -139,10 +164,15 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { func TestSynchronizer_AddEventsOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 4 - events := buildEvents(4) + events := buildServerUpdateEvents(1) rateLimiter := &mocks.MockRateLimiter{} - synchronizer, err := NewSynchronizer(defaultSettings("https://localhost:8080"), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings("https://localhost:8080"), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -151,7 +181,10 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvents(events) + for i := 0; i < 4; i++ { + synchronizer.AddEvent(buildServiceUpdateEvent(i)) + } + actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) @@ -161,7 +194,7 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { func TestSynchronizer_AddEventsManyHosts(t *testing.T) { t.Parallel() const eventCount = 4 - events := buildEvents(eventCount) + events := buildServerUpdateEvents(eventCount) rateLimiter := &mocks.MockRateLimiter{} hosts := []string{ @@ -170,9 +203,14 @@ func TestSynchronizer_AddEventsManyHosts(t *testing.T) { "https://localhost:8082", } - expectedEventCount := eventCount * len(hosts) + expectedEventCount := 4 - synchronizer, err := NewSynchronizer(defaultSettings(hosts...), rateLimiter) + synchronizer, err := NewSynchronizer( + defaultSettings(hosts...), + rateLimiter, + &fakeTranslator{events, nil}, + newFakeServicesLister(defaultService()), + ) if err != nil { t.Fatalf(`should have been no error, %v`, err) } @@ -181,14 +219,28 @@ func TestSynchronizer_AddEventsManyHosts(t *testing.T) { t.Fatal("should have an Synchronizer instance") } - synchronizer.AddEvents(events) + for i := 0; i < eventCount; i++ { + synchronizer.AddEvent(buildServiceUpdateEvent(i)) + } + actualEventCount := rateLimiter.Len() if actualEventCount != expectedEventCount { t.Fatalf(`expected %v events, got %v`, expectedEventCount, actualEventCount) } } -func buildEvents(count int) core.ServerUpdateEvents { +func buildServiceUpdateEvent(serviceID int) core.Event { + return core.Event{ + Service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("test-service%d", serviceID), + Namespace: "test-namespace", + }, + }, + } +} + +func buildServerUpdateEvents(count int) core.ServerUpdateEvents { events := make(core.ServerUpdateEvents, count) for i := 0; i < count; i++ { events[i] = &core.ServerUpdateEvent{ @@ -218,3 +270,53 @@ func defaultSettings(nginxHosts ...string) configuration.Settings { }, } } + +type fakeTranslator struct { + events core.ServerUpdateEvents + err error +} + +func (t *fakeTranslator) Translate(event *core.Event) (core.ServerUpdateEvents, error) { + return t.events, t.err +} + +func newFakeServicesLister(list ...*v1.Service) corelisters.ServiceLister { + return &servicesLister{ + list: list, + } +} + +type servicesLister struct { + list []*v1.Service + err error +} + +func (l *servicesLister) List(selector labels.Selector) (ret []*v1.Service, err error) { + return l.list, l.err +} + +func (l *servicesLister) Get(name string) (*v1.Service, error) { + for _, service := range l.list { + if service.Name == name { + return service, nil + } + } + + return nil, nil +} + +func (l *servicesLister) Services(name string) corelisters.ServiceNamespaceLister { + return l +} + +func defaultService() *v1.Service { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-service", + Labels: map[string]string{"kubernetes.io/service-name": "default-service"}, + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeNodePort, + }, + } +} diff --git a/test/mocks/mock_handler.go b/test/mocks/mock_handler.go deleted file mode 100644 index f144cea..0000000 --- a/test/mocks/mock_handler.go +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package mocks - -import ( - "context" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -type MockHandler struct{} - -func (h *MockHandler) AddRateLimitedEvent(_ *core.Event) { -} - -func (h *MockHandler) Initialize() { -} - -func (h *MockHandler) Run(ctx context.Context) { -} - -func (h *MockHandler) ShutDown() { -} From 71e9c1552b987caa85400382f7e11000472d81fc Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 11 Dec 2024 10:31:48 -0700 Subject: [PATCH 098/136] NLB-5933 Added error group to the main routine Even though this is not really modifying the application's behavior right now (the watcher and synchronizer were only returning when the context was cancelled) it is a nicer framework for handling the errors of modules launched in separate goroutines if we do choose to make any of the NLK modules return errors. --- cmd/nginx-loadbalancer-kubernetes/main.go | 14 +++++++------- go.mod | 1 + go.sum | 2 ++ internal/synchronization/synchronizer.go | 10 ++++++---- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 501d7c4..5aba57f 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -17,6 +17,7 @@ import ( "github.com/nginxinc/kubernetes-nginx-ingress/internal/synchronization" "github.com/nginxinc/kubernetes-nginx-ingress/internal/translation" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" + "golang.org/x/sync/errgroup" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -80,18 +81,17 @@ func run() error { } } - go synchronizer.Run(ctx.Done()) + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { return synchronizer.Run(ctx) }) probeServer := probation.NewHealthServer() probeServer.Start() - err = watcher.Run(ctx) - if err != nil { - return fmt.Errorf(`error occurred running watcher: %w`, err) - } + g.Go(func() error { return watcher.Run(ctx) }) - <-ctx.Done() - return nil + err = g.Wait() + return err } func initializeLogger(logLevel string) { diff --git a/go.mod b/go.mod index 94a1319..cc4fe63 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 + golang.org/x/sync v0.6.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 diff --git a/go.sum b/go.sum index 1433805..bd58f37 100644 --- a/go.sum +++ b/go.sum @@ -180,6 +180,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index fef9724..0d70215 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -6,6 +6,7 @@ package synchronization import ( + "context" "errors" "fmt" "log/slog" @@ -27,7 +28,7 @@ type Interface interface { AddEvent(event core.Event) // Run starts the synchronizer. - Run(stopCh <-chan struct{}) + Run(ctx context.Context) error // ShutDown shuts down the synchronizer. ShutDown() @@ -87,14 +88,15 @@ func (s *Synchronizer) AddEvent(event core.Event) { } // Run starts the Synchronizer, spins up Goroutines to process events, and waits for a stop signal. -func (s *Synchronizer) Run(stopCh <-chan struct{}) { +func (s *Synchronizer) Run(ctx context.Context) error { slog.Debug(`Synchronizer::Run`) for i := 0; i < s.settings.Synchronizer.Threads; i++ { - go wait.Until(s.worker, 0, stopCh) + go wait.Until(s.worker, 0, ctx.Done()) } - <-stopCh + <-ctx.Done() + return nil } // ShutDown stops the Synchronizer and shuts down the event queue From 1de0c2c3079556df747270b7ea379f441b87eb6c Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 17 Dec 2024 16:24:18 -0700 Subject: [PATCH 099/136] NLB-5933 Remove PreviousService from core event This field was not being used anywhere, so it seems better to get rid of it and avoid the hassle of keeping it accurate. --- internal/core/event.go | 10 +++------- internal/core/event_test.go | 7 +------ internal/observation/watcher.go | 26 +++++++++---------------- internal/translation/translator_test.go | 4 +--- 4 files changed, 14 insertions(+), 33 deletions(-) diff --git a/internal/core/event.go b/internal/core/event.go index 16d5d94..c32511e 100644 --- a/internal/core/event.go +++ b/internal/core/event.go @@ -29,16 +29,12 @@ type Event struct { // Service represents the service object in its current state Service *v1.Service - - // PreviousService represents the service object in its previous state - PreviousService *v1.Service } // NewEvent factory method to create a new Event -func NewEvent(eventType EventType, service *v1.Service, previousService *v1.Service) Event { +func NewEvent(eventType EventType, service *v1.Service) Event { return Event{ - Type: eventType, - Service: service, - PreviousService: previousService, + Type: eventType, + Service: service, } } diff --git a/internal/core/event_test.go b/internal/core/event_test.go index f0184fb..09724cf 100644 --- a/internal/core/event_test.go +++ b/internal/core/event_test.go @@ -10,9 +10,8 @@ func TestNewEvent(t *testing.T) { t.Parallel() expectedType := Created expectedService := &v1.Service{} - expectedPreviousService := &v1.Service{} - event := NewEvent(expectedType, expectedService, expectedPreviousService) + event := NewEvent(expectedType, expectedService) if event.Type != expectedType { t.Errorf("expected Created, got %v", event.Type) @@ -21,8 +20,4 @@ func TestNewEvent(t *testing.T) { if event.Service != expectedService { t.Errorf("expected service, got %#v", event.Service) } - - if event.PreviousService != expectedPreviousService { - t.Errorf("expected previous service, got %#v", event.PreviousService) - } } diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 6c8977e..0df49fe 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -113,8 +113,7 @@ func (w *Watcher) buildNodesEventHandlerForAdd() func(interface{}) { return func(obj interface{}) { slog.Debug("received node add event") for _, service := range w.register.listServices() { - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -125,8 +124,7 @@ func (w *Watcher) buildNodesEventHandlerForUpdate() func(interface{}, interface{ return func(previous, updated interface{}) { slog.Debug("received node update event") for _, service := range w.register.listServices() { - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -137,8 +135,7 @@ func (w *Watcher) buildNodesEventHandlerForDelete() func(interface{}) { return func(obj interface{}) { slog.Debug("received node delete event") for _, service := range w.register.listServices() { - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -160,8 +157,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForAdd() func(interface{}) { return } - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -182,8 +178,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForUpdate() func(interface{}, i return } - var previousService *v1.Service - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } @@ -204,8 +199,7 @@ func (w *Watcher) buildEndpointSlicesEventHandlerForDelete() func(interface{}) { return } - var previousService *v1.Service - e := core.NewEvent(core.Deleted, service, previousService) + e := core.NewEvent(core.Deleted, service) w.synchronizer.AddEvent(e) } } @@ -222,8 +216,7 @@ func (w *Watcher) buildServiceEventHandlerForAdd() func(interface{}) { w.register.addOrUpdateService(service) - var previousService *v1.Service - e := core.NewEvent(core.Created, service, previousService) + e := core.NewEvent(core.Created, service) w.synchronizer.AddEvent(e) } } @@ -240,8 +233,7 @@ func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { w.register.removeService(service) - var previousService *v1.Service - e := core.NewEvent(core.Deleted, service, previousService) + e := core.NewEvent(core.Deleted, service) w.synchronizer.AddEvent(e) } } @@ -266,7 +258,7 @@ func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interfac w.register.addOrUpdateService(service) - e := core.NewEvent(core.Updated, service, previousService) + e := core.NewEvent(core.Updated, service) w.synchronizer.AddEvent(e) } } diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index c6d42f5..11fa22c 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -1324,9 +1324,7 @@ func buildUpdatedEvent(service *v1.Service) core.Event { } func buildEvent(eventType core.EventType, service *v1.Service) core.Event { - previousService := defaultService(service.Spec.Type) - - event := core.NewEvent(eventType, service, previousService) + event := core.NewEvent(eventType, service) event.Service.Name = "default-service" return event } From a504336086064a553fdc0bcfd78593b153005b2d Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 19 Dec 2024 10:32:29 -0700 Subject: [PATCH 100/136] NLB-5933 Handle 404 errors when deleting upstream servers If a user removes a upstream server from their nginx configuration and then deletes the service associated with that upstream from their AKS cluster, we should handle the resulting 404s on the upstream servers and forget about the workitem, rather than retrying the updates indefinitely. Checking the error for a substring is not ideal as a way of handling the error, but the go plus client gives us no option here. --- internal/synchronization/synchronizer.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 0d70215..63b032a 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/communication" @@ -246,11 +247,17 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv return fmt.Errorf(`error occurred creating the border client: %w`, err) } - if err = borderClient.Delete(serverUpdateEvent); err != nil { + err = borderClient.Update(serverUpdateEvent) + + switch { + case err == nil: + return nil + // checking the string is not ideal, but the plus client gives us no option + case strings.Contains(err.Error(), "status=404"): + return nil + default: return fmt.Errorf(`error occurred deleting the %s upstream servers: %w`, serverUpdateEvent.ClientType, err) } - - return nil } // handleNextServiceEvent pulls a service from the event queue and feeds it to From dec267fb0d41ca96df1c003c575532d76a956566 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 19 Dec 2024 14:58:53 -0700 Subject: [PATCH 101/136] NLB-5933 Bumped version to 0.8.2 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 6f4eebd..100435b 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.1 +0.8.2 From 82e91f88fe3d258c091cf5a46731f31a69773127 Mon Sep 17 00:00:00 2001 From: sarna Date: Sat, 4 Jan 2025 18:22:29 +0530 Subject: [PATCH 102/136] Upgrade the indirect net dep The golang.org/x/net package has a high CVE. --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index cc4fe63..502888d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/nginxinc/nginx-plus-go-client v1.2.2 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.10.0 k8s.io/api v0.26.0 k8s.io/apimachinery v0.26.0 k8s.io/client-go v0.26.0 @@ -55,11 +55,11 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.23.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index bd58f37..26b1b01 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= @@ -180,8 +180,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -190,18 +190,18 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 3aa3a4cc2deb9287ecec914753971da58c6c2358 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 9 Jan 2025 13:27:56 -0700 Subject: [PATCH 103/136] Upgrade nginx-plus-go-client to v2.2.0 This update brings changes that cause the client to apply all upstream server updates and not to return on first error. The upgrade necessitates changing the border client interfaces to match the new method signatures of the plus go client. As a result, I've threaded the context through various methods which did not require it before. --- go.mod | 6 ++-- go.sum | 4 +-- internal/application/border_client.go | 5 +-- .../application/nginx_client_interface.go | 14 ++++++-- .../application/nginx_http_border_client.go | 11 +++--- .../nginx_http_border_client_test.go | 9 ++--- .../application/nginx_stream_border_client.go | 11 +++--- .../nginx_stream_border_client_test.go | 9 ++--- internal/application/null_border_client.go | 5 +-- .../application/null_border_client_test.go | 9 +++-- internal/synchronization/synchronizer.go | 36 +++++++++---------- test/mocks/mock_nginx_plus_client.go | 12 +++++-- 12 files changed, 77 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 502888d..512dc36 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.21.2 +go 1.22.6 -toolchain go1.21.4 +toolchain go1.23.4 require ( - github.com/nginxinc/nginx-plus-go-client v1.2.2 + github.com/nginx/nginx-plus-go-client/v2 v2.2.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 golang.org/x/sync v0.10.0 diff --git a/go.sum b/go.sum index 26b1b01..cae00ce 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginxinc/nginx-plus-go-client v1.2.2 h1:sl7HqNDDZq2EVu0eQQVoZ6PKYGa4h2dB/Qr5Ib0YKGw= -github.com/nginxinc/nginx-plus-go-client v1.2.2/go.mod h1:n8OFLzrJulJ2fur28Cwa1Qp5DZNS2VicLV+Adt30LQ4= +github.com/nginx/nginx-plus-go-client/v2 v2.2.0 h1:qwhx4fF/pq+h72/nE+o+XSH5mZmDU/R8fwim6VcZ8cM= +github.com/nginx/nginx-plus-go-client/v2 v2.2.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= diff --git a/internal/application/border_client.go b/internal/application/border_client.go index e4eded8..8bca843 100644 --- a/internal/application/border_client.go +++ b/internal/application/border_client.go @@ -6,6 +6,7 @@ package application import ( + "context" "fmt" "log/slog" @@ -14,8 +15,8 @@ import ( // Interface defines the functions required to implement a Border Client. type Interface interface { - Update(*core.ServerUpdateEvent) error - Delete(*core.ServerUpdateEvent) error + Update(context.Context, *core.ServerUpdateEvent) error + Delete(context.Context, *core.ServerUpdateEvent) error } // BorderClient defines any state need by the Border Client. diff --git a/internal/application/nginx_client_interface.go b/internal/application/nginx_client_interface.go index 31a7b94..1a60c5e 100644 --- a/internal/application/nginx_client_interface.go +++ b/internal/application/nginx_client_interface.go @@ -5,25 +5,33 @@ package application -import nginxClient "github.com/nginxinc/nginx-plus-go-client/client" +import ( + "context" + + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" +) + +var _ NginxClientInterface = (*nginxClient.NginxClient)(nil) // NginxClientInterface defines the functions used on the NGINX Plus client, // abstracting away the full details of that client. type NginxClientInterface interface { // DeleteStreamServer is used by the NginxStreamBorderClient. - DeleteStreamServer(upstream string, server string) error + DeleteStreamServer(ctx context.Context, upstream string, server string) error // UpdateStreamServers is used by the NginxStreamBorderClient. UpdateStreamServers( + ctx context.Context, upstream string, servers []nginxClient.StreamUpstreamServer, ) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) // DeleteHTTPServer is used by the NginxHTTPBorderClient. - DeleteHTTPServer(upstream string, server string) error + DeleteHTTPServer(ctx context.Context, upstream string, server string) error // UpdateHTTPServers is used by the NginxHTTPBorderClient. UpdateHTTPServers( + ctx context.Context, upstream string, servers []nginxClient.UpstreamServer, ) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) diff --git a/internal/application/nginx_http_border_client.go b/internal/application/nginx_http_border_client.go index e9e754a..4de147e 100644 --- a/internal/application/nginx_http_border_client.go +++ b/internal/application/nginx_http_border_client.go @@ -7,10 +7,11 @@ package application import ( + "context" "fmt" + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) // NginxHttpBorderClient implements the BorderClient interface for HTTP upstreams. @@ -32,9 +33,9 @@ func NewNginxHTTPBorderClient(client interface{}) (Interface, error) { } // Update manages the Upstream servers for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHTTPBorderClient) Update(event *core.ServerUpdateEvent) error { +func (hbc *NginxHTTPBorderClient) Update(ctx context.Context, event *core.ServerUpdateEvent) error { httpUpstreamServers := asNginxHTTPUpstreamServers(event.UpstreamServers) - _, _, _, err := hbc.nginxClient.UpdateHTTPServers(event.UpstreamName, httpUpstreamServers) + _, _, _, err := hbc.nginxClient.UpdateHTTPServers(ctx, event.UpstreamName, httpUpstreamServers) if err != nil { return fmt.Errorf(`error occurred updating the nginx+ upstream server: %w`, err) } @@ -43,8 +44,8 @@ func (hbc *NginxHTTPBorderClient) Update(event *core.ServerUpdateEvent) error { } // Delete deletes the Upstream server for the Upstream Name given in the ServerUpdateEvent. -func (hbc *NginxHTTPBorderClient) Delete(event *core.ServerUpdateEvent) error { - err := hbc.nginxClient.DeleteHTTPServer(event.UpstreamName, event.UpstreamServers[0].Host) +func (hbc *NginxHTTPBorderClient) Delete(ctx context.Context, event *core.ServerUpdateEvent) error { + err := hbc.nginxClient.DeleteHTTPServer(ctx, event.UpstreamName, event.UpstreamServers[0].Host) if err != nil { return fmt.Errorf(`error occurred deleting the nginx+ upstream server: %w`, err) } diff --git a/internal/application/nginx_http_border_client_test.go b/internal/application/nginx_http_border_client_test.go index 039b4ec..7a97206 100644 --- a/internal/application/nginx_http_border_client_test.go +++ b/internal/application/nginx_http_border_client_test.go @@ -9,6 +9,7 @@ package application import ( + "context" "testing" ) @@ -20,7 +21,7 @@ func TestHttpBorderClient_Delete(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -38,7 +39,7 @@ func TestHttpBorderClient_Update(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -65,7 +66,7 @@ func TestHttpBorderClient_DeleteReturnsError(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) @@ -80,7 +81,7 @@ func TestHttpBorderClient_UpdateReturnsError(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index 19982b4..238a22b 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -7,10 +7,11 @@ package application import ( + "context" "fmt" + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - nginxClient "github.com/nginxinc/nginx-plus-go-client/client" ) // NginxStreamBorderClient implements the BorderClient interface for stream upstreams. @@ -32,9 +33,9 @@ func NewNginxStreamBorderClient(client interface{}) (Interface, error) { } // Update manages the Upstream servers for the Upstream Name given in the ServerUpdateEvent. -func (tbc *NginxStreamBorderClient) Update(event *core.ServerUpdateEvent) error { +func (tbc *NginxStreamBorderClient) Update(ctx context.Context, event *core.ServerUpdateEvent) error { streamUpstreamServers := asNginxStreamUpstreamServers(event.UpstreamServers) - _, _, _, err := tbc.nginxClient.UpdateStreamServers(event.UpstreamName, streamUpstreamServers) + _, _, _, err := tbc.nginxClient.UpdateStreamServers(ctx, event.UpstreamName, streamUpstreamServers) if err != nil { return fmt.Errorf(`error occurred updating the nginx+ upstream server: %w`, err) } @@ -43,8 +44,8 @@ func (tbc *NginxStreamBorderClient) Update(event *core.ServerUpdateEvent) error } // Delete deletes the Upstream server for the Upstream Name given in the ServerUpdateEvent. -func (tbc *NginxStreamBorderClient) Delete(event *core.ServerUpdateEvent) error { - err := tbc.nginxClient.DeleteStreamServer(event.UpstreamName, event.UpstreamServers[0].Host) +func (tbc *NginxStreamBorderClient) Delete(ctx context.Context, event *core.ServerUpdateEvent) error { + err := tbc.nginxClient.DeleteStreamServer(ctx, event.UpstreamName, event.UpstreamServers[0].Host) if err != nil { return fmt.Errorf(`error occurred deleting the nginx+ upstream server: %w`, err) } diff --git a/internal/application/nginx_stream_border_client_test.go b/internal/application/nginx_stream_border_client_test.go index c86a776..cf4d302 100644 --- a/internal/application/nginx_stream_border_client_test.go +++ b/internal/application/nginx_stream_border_client_test.go @@ -7,6 +7,7 @@ package application import ( + "context" "testing" ) @@ -18,7 +19,7 @@ func TestTcpBorderClient_Delete(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -36,7 +37,7 @@ func TestTcpBorderClient_Update(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err != nil { t.Fatalf(`error occurred deleting the nginx+ upstream server: %v`, err) } @@ -63,7 +64,7 @@ func TestTcpBorderClient_DeleteReturnsError(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Delete(event) + err = borderClient.Delete(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) @@ -78,7 +79,7 @@ func TestTcpBorderClient_UpdateReturnsError(t *testing.T) { t.Fatalf(`error occurred creating a new border client: %v`, err) } - err = borderClient.Update(event) + err = borderClient.Update(context.Background(), event) if err == nil { t.Fatalf(`expected an error to occur when deleting the nginx+ upstream server`) diff --git a/internal/application/null_border_client.go b/internal/application/null_border_client.go index 295ca62..dc4467d 100644 --- a/internal/application/null_border_client.go +++ b/internal/application/null_border_client.go @@ -6,6 +6,7 @@ package application import ( + "context" "log/slog" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" @@ -22,13 +23,13 @@ func NewNullBorderClient() (Interface, error) { } // Update logs a Warning. It is, after all, a NullObject Pattern implementation. -func (nbc *NullBorderClient) Update(_ *core.ServerUpdateEvent) error { +func (nbc *NullBorderClient) Update(_ context.Context, _ *core.ServerUpdateEvent) error { slog.Warn("NullBorderClient.Update called") return nil } // Delete logs a Warning. It is, after all, a NullObject Pattern implementation. -func (nbc *NullBorderClient) Delete(_ *core.ServerUpdateEvent) error { +func (nbc *NullBorderClient) Delete(_ context.Context, _ *core.ServerUpdateEvent) error { slog.Warn("NullBorderClient.Delete called") return nil } diff --git a/internal/application/null_border_client_test.go b/internal/application/null_border_client_test.go index f973949..01d9fe2 100644 --- a/internal/application/null_border_client_test.go +++ b/internal/application/null_border_client_test.go @@ -5,12 +5,15 @@ package application -import "testing" +import ( + "context" + "testing" +) func TestNullBorderClient_Delete(t *testing.T) { t.Parallel() client := NullBorderClient{} - err := client.Delete(nil) + err := client.Delete(context.Background(), nil) if err != nil { t.Errorf(`expected no error deleting border client, got: %v`, err) } @@ -19,7 +22,7 @@ func TestNullBorderClient_Delete(t *testing.T) { func TestNullBorderClient_Update(t *testing.T) { t.Parallel() client := NullBorderClient{} - err := client.Update(nil) + err := client.Update(context.Background(), nil) if err != nil { t.Errorf(`expected no error updating border client, got: %v`, err) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 63b032a..d45277b 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -12,11 +12,11 @@ import ( "log/slog" "strings" + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" "github.com/nginxinc/kubernetes-nginx-ingress/internal/communication" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" - nginxClient "github.com/nginxinc/nginx-plus-go-client/client" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" corelisters "k8s.io/client-go/listers/core/v1" @@ -92,8 +92,15 @@ func (s *Synchronizer) AddEvent(event core.Event) { func (s *Synchronizer) Run(ctx context.Context) error { slog.Debug(`Synchronizer::Run`) + // worker is the main message loop + worker := func() { + slog.Debug(`Synchronizer::worker`) + for s.handleNextServiceEvent(ctx) { + } + } + for i := 0; i < s.settings.Synchronizer.Threads; i++ { - go wait.Until(s.worker, 0, ctx.Done()) + go wait.Until(worker, 0, ctx.Done()) } <-ctx.Done() @@ -148,7 +155,7 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se // handleServiceEvent gets the latest state for the service from the shared // informer cache, translates the service event into server update events and // dispatches these events to the proper handler function. -func (s *Synchronizer) handleServiceEvent(key ServiceKey) (err error) { +func (s *Synchronizer) handleServiceEvent(ctx context.Context, key ServiceKey) (err error) { logger := slog.With("service", key) logger.Debug(`Synchronizer::handleServiceEvent`) @@ -191,11 +198,11 @@ func (s *Synchronizer) handleServiceEvent(key ServiceKey) (err error) { for _, evt := range events { switch event.Type { case core.Created, core.Updated: - if handleErr := s.handleCreatedUpdatedEvent(evt); handleErr != nil { + if handleErr := s.handleCreatedUpdatedEvent(ctx, evt); handleErr != nil { err = errors.Join(err, handleErr) } case core.Deleted: - if handleErr := s.handleDeletedEvent(evt); handleErr != nil { + if handleErr := s.handleDeletedEvent(ctx, evt); handleErr != nil { err = errors.Join(err, handleErr) } default: @@ -219,7 +226,7 @@ func (s *Synchronizer) handleServiceEvent(key ServiceKey) (err error) { } // handleCreatedUpdatedEvent handles events of type Created or Updated. -func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { +func (s *Synchronizer) handleCreatedUpdatedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { slog.Debug(`Synchronizer::handleCreatedUpdatedEvent`, "eventID", serverUpdateEvent.ID) var err error @@ -229,7 +236,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU return fmt.Errorf(`error occurred creating the border client: %w`, err) } - if err = borderClient.Update(serverUpdateEvent); err != nil { + if err = borderClient.Update(ctx, serverUpdateEvent); err != nil { return fmt.Errorf(`error occurred updating the %s upstream servers: %w`, serverUpdateEvent.ClientType, err) } @@ -237,7 +244,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(serverUpdateEvent *core.ServerU } // handleDeletedEvent handles events of type Deleted. -func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEvent) error { +func (s *Synchronizer) handleDeletedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { slog.Debug(`Synchronizer::handleDeletedEvent`, "eventID", serverUpdateEvent.ID) var err error @@ -247,7 +254,7 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv return fmt.Errorf(`error occurred creating the border client: %w`, err) } - err = borderClient.Update(serverUpdateEvent) + err = borderClient.Update(ctx, serverUpdateEvent) switch { case err == nil: @@ -262,7 +269,7 @@ func (s *Synchronizer) handleDeletedEvent(serverUpdateEvent *core.ServerUpdateEv // handleNextServiceEvent pulls a service from the event queue and feeds it to // the service event handler with retry logic -func (s *Synchronizer) handleNextServiceEvent() bool { +func (s *Synchronizer) handleNextServiceEvent(ctx context.Context) bool { slog.Debug(`Synchronizer::handleNextServiceEvent`) svc, quit := s.eventQueue.Get() @@ -278,18 +285,11 @@ func (s *Synchronizer) handleNextServiceEvent() bool { return true } - s.withRetry(s.handleServiceEvent(key), key) + s.withRetry(s.handleServiceEvent(ctx, key), key) return true } -// worker is the main message loop -func (s *Synchronizer) worker() { - slog.Debug(`Synchronizer::worker`) - for s.handleNextServiceEvent() { - } -} - // withRetry handles errors from the event handler and requeues events that fail func (s *Synchronizer) withRetry(err error, key ServiceKey) { slog.Debug("Synchronizer::withRetry") diff --git a/test/mocks/mock_nginx_plus_client.go b/test/mocks/mock_nginx_plus_client.go index 00b560e..991ab08 100644 --- a/test/mocks/mock_nginx_plus_client.go +++ b/test/mocks/mock_nginx_plus_client.go @@ -5,7 +5,11 @@ package mocks -import nginxClient "github.com/nginxinc/nginx-plus-go-client/client" +import ( + "context" + + nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" +) type MockNginxClient struct { CalledFunctions map[string]bool @@ -26,7 +30,7 @@ func NewErroringMockClient(err error) *MockNginxClient { } } -func (m MockNginxClient) DeleteStreamServer(_ string, _ string) error { +func (m MockNginxClient) DeleteStreamServer(_ context.Context, _ string, _ string) error { m.CalledFunctions["DeleteStreamServer"] = true if m.Error != nil { @@ -37,6 +41,7 @@ func (m MockNginxClient) DeleteStreamServer(_ string, _ string) error { } func (m MockNginxClient) UpdateStreamServers( + _ context.Context, _ string, _ []nginxClient.StreamUpstreamServer, ) ([]nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, []nginxClient.StreamUpstreamServer, error) { @@ -49,7 +54,7 @@ func (m MockNginxClient) UpdateStreamServers( return nil, nil, nil, nil } -func (m MockNginxClient) DeleteHTTPServer(_ string, _ string) error { +func (m MockNginxClient) DeleteHTTPServer(_ context.Context, _ string, _ string) error { m.CalledFunctions["DeleteHTTPServer"] = true if m.Error != nil { @@ -60,6 +65,7 @@ func (m MockNginxClient) DeleteHTTPServer(_ string, _ string) error { } func (m MockNginxClient) UpdateHTTPServers( + _ context.Context, _ string, _ []nginxClient.UpstreamServer, ) ([]nginxClient.UpstreamServer, []nginxClient.UpstreamServer, []nginxClient.UpstreamServer, error) { From 5fd552e07ee612ee5bd4ba0cc048bb1f1545b755 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 9 Jan 2025 15:36:19 -0700 Subject: [PATCH 104/136] Bumped version to 0.8.3 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 100435b..ee94dd8 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.2 +0.8.3 From 8827dd1595c2e1a42f64125eb554d531a34f05c6 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 21 Jan 2025 16:09:41 -0700 Subject: [PATCH 105/136] NLB-6070 Removal of service annotation is equivalent to service deletion We interpret the removal of the nginxaas service annotation as a sign from the user that they no longer wish us to route traffic to the upstreams formerly associated with theservice. It is as if they were at the starting point of not yet having configured NLK to monitor a specific kubernetes service. When we see that the nginxaas service annotation has been removed, we delete all upstream servers from the upstreams associated with the service. --- internal/observation/watcher.go | 4 +++- internal/synchronization/cache.go | 21 +++++++++++++++------ internal/synchronization/synchronizer.go | 18 ++++++++++++++---- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/internal/observation/watcher.go b/internal/observation/watcher.go index 0df49fe..9711347 100644 --- a/internal/observation/watcher.go +++ b/internal/observation/watcher.go @@ -243,12 +243,14 @@ func (w *Watcher) buildServiceEventHandlerForDelete() func(interface{}) { func (w *Watcher) buildServiceEventHandlerForUpdate() func(interface{}, interface{}) { slog.Info("Watcher::buildServiceEventHandlerForUpdate") return func(previous, updated interface{}) { - // TODO NLB-5435 Check for user removing annotation and send delete request to dataplane API previousService := previous.(*v1.Service) service := updated.(*v1.Service) if w.isDesiredService(previousService) && !w.isDesiredService(service) { + slog.Info("Watcher::service annotation removed", "serviceName", service.Name) w.register.removeService(previousService) + e := core.NewEvent(core.Deleted, previousService) + w.synchronizer.AddEvent(e) return } diff --git a/internal/synchronization/cache.go b/internal/synchronization/cache.go index 6c3f164..14effb9 100644 --- a/internal/synchronization/cache.go +++ b/internal/synchronization/cache.go @@ -2,6 +2,7 @@ package synchronization import ( "sync" + "time" v1 "k8s.io/api/core/v1" ) @@ -11,23 +12,31 @@ import ( // caller can access the spec of the deleted service for cleanup. type cache struct { mu sync.RWMutex - store map[ServiceKey]*v1.Service + store map[ServiceKey]service +} + +type service struct { + service *v1.Service + // removedAt indicates when the service was removed from NGINXaaS + // monitoring. A zero time indicates that the service is still actively + // being monitored by NGINXaaS. + removedAt time.Time } func newCache() *cache { return &cache{ - store: make(map[ServiceKey]*v1.Service), + store: make(map[ServiceKey]service), } } -func (s *cache) get(key ServiceKey) (*v1.Service, bool) { +func (s *cache) get(key ServiceKey) (service, bool) { s.mu.RLock() defer s.mu.RUnlock() - service, ok := s.store[key] - return service, ok + svc, ok := s.store[key] + return svc, ok } -func (s *cache) add(key ServiceKey, service *v1.Service) { +func (s *cache) add(key ServiceKey, service service) { s.mu.Lock() defer s.mu.Unlock() s.store[key] = service diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index d45277b..82b3071 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -11,6 +11,7 @@ import ( "fmt" "log/slog" "strings" + "time" nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" "github.com/nginxinc/kubernetes-nginx-ingress/internal/application" @@ -84,7 +85,12 @@ func (s *Synchronizer) AddEvent(event core.Event) { } key := ServiceKey{Name: event.Service.Name, Namespace: event.Service.Namespace} - s.cache.add(key, event.Service) + var deletedAt time.Time + if event.Type == core.Deleted { + deletedAt = time.Now() + } + + s.cache.add(key, service{event.Service, deletedAt}) s.eventQueue.AddRateLimited(key) } @@ -162,6 +168,8 @@ func (s *Synchronizer) handleServiceEvent(ctx context.Context, key ServiceKey) ( // if a service exists in the shared informer cache, we can assume that we need to update it event := core.Event{Type: core.Updated} + cachedService, exists := s.cache.get(key) + namespaceLister := s.serviceLister.Services(key.Namespace) k8sService, err := namespaceLister.Get(key.Name) switch { @@ -169,16 +177,18 @@ func (s *Synchronizer) handleServiceEvent(ctx context.Context, key ServiceKey) ( // gather the last known state of the service so we can delete its // upstream servers case err != nil && apierrors.IsNotFound(err): - service, ok := s.cache.get(key) - if !ok { + if !exists { logger.Warn(`Synchronizer::handleServiceEvent: no information could be gained about service`) return nil } // no matter what type the cached event has, the service no longer exists, so the type is Deleted event.Type = core.Deleted - event.Service = service + event.Service = cachedService.service case err != nil: return err + case exists && !cachedService.removedAt.IsZero(): + event.Type = core.Deleted + event.Service = cachedService.service default: event.Service = k8sService } From e839b5c70a15511aafde4e036273e76310433bce Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 21 Jan 2025 16:27:06 -0700 Subject: [PATCH 106/136] NLB-6070 bumped version to 0.8.4 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index ee94dd8..b60d719 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.3 +0.8.4 From da4caf026bb97d76bd99ad4563f8c55d22f4e07f Mon Sep 17 00:00:00 2001 From: sarna Date: Wed, 29 Jan 2025 16:25:19 -0800 Subject: [PATCH 107/136] Use upstream SAST template We were using the deprecated internal SAST template which was retired a while back: https://gitlab.com/f5/nginx/tools/devops-utils/-/merge_requests/204. There was a change made to the upstream template that is included in the internal template which breaks the internal template as there were some jobs that were removed from the upstream. This commit moves us over to use the official upstream template. --- .gitlab-ci.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 375a262..04dffff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,9 +9,7 @@ include: - project: "f5/nginx/tools/devops-utils" file: "include/devops-whitesource.yaml" ref: "master" - - project: "f5/nginx/tools/devops-utils" - file: "/include/gitlab-sast.yml" - ref: "master" + - template: Jobs/SAST.gitlab-ci.yml - template: Jobs/Container-Scanning.gitlab-ci.yml stages: @@ -163,15 +161,6 @@ whitesource-scan: - ${CI_PROJECT_DIR}/whitesource/ expire_in: 1 weeks -################################################################################ -# GitLab SAST security scanning -# CISO requires that all products be scanned statically for vulnerabilities. -# For more information, please see: -# https://gitlab.com/f5/nginx/tools/devops-utils/-/blob/master/include/gitlab-sast.md -# NOTE: please do not alter, change, modify, or overwrite job rules, job -# variables, or runtime states in order to maintain compliance to CISO -# requirements. -################################################################################ sast: stage: lint+test+build From 751ff3f851fd19e2fad09e34fe111bf783f51b32 Mon Sep 17 00:00:00 2001 From: Dylan WAY Date: Wed, 29 Jan 2025 14:21:56 -0700 Subject: [PATCH 108/136] NLB-5828: update nginx-plus-go-client version Pulls in the latest nginx-plus-go-client to make use of some optimizations in the UpdateStreamServers and UpdateHTTPServers calls. --- go.mod | 2 +- go.sum | 4 ++-- version | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 512dc36..f7c3dc8 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ go 1.22.6 toolchain go1.23.4 require ( - github.com/nginx/nginx-plus-go-client/v2 v2.2.0 + github.com/nginx/nginx-plus-go-client/v2 v2.3.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 golang.org/x/sync v0.10.0 diff --git a/go.sum b/go.sum index cae00ce..505b7f6 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginx/nginx-plus-go-client/v2 v2.2.0 h1:qwhx4fF/pq+h72/nE+o+XSH5mZmDU/R8fwim6VcZ8cM= -github.com/nginx/nginx-plus-go-client/v2 v2.2.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= +github.com/nginx/nginx-plus-go-client/v2 v2.3.0 h1:ciKh1lwadNzUaOGjLcKWu/BGigASxU6p7v/6US71fhA= +github.com/nginx/nginx-plus-go-client/v2 v2.3.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= diff --git a/version b/version index b60d719..7ada0d3 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.4 +0.8.5 From 1c47001fe1374fc6d97d0cfcb637a36580c002fe Mon Sep 17 00:00:00 2001 From: sarna Date: Tue, 4 Feb 2025 12:08:07 -0800 Subject: [PATCH 109/136] Release NLK 1.0.0 This marks the official GA release for NLK. The updated semver should help communicate the readiness of the integration and the product to the customers. --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 7ada0d3..3eefcb9 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.5 +1.0.0 From 5ce472a8cbc15f5cbac59459ea36f1ea8700f1a3 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 10 Feb 2025 14:43:17 -0700 Subject: [PATCH 110/136] NLB-6230 Handle services of type LoadBalancer NLK listens for changes to any kubernetes service of type LoadBalancer and updates the NGINXaaS upstreams with the external IPs and ports of the LB service. We are using the external IPs and not the node IPs so as to provide seamless plug-and-play functionality with the customer's networking framework. --- internal/translation/translator.go | 42 ++++- internal/translation/translator_test.go | 210 +++++++++++++++++++----- 2 files changed, 210 insertions(+), 42 deletions(-) diff --git a/internal/translation/translator.go b/internal/translation/translator.go index 117c0b9..8491a66 100644 --- a/internal/translation/translator.go +++ b/internal/translation/translator.go @@ -56,6 +56,8 @@ func (t *Translator) buildServerUpdateEvents(ports []v1.ServicePort, event *core return t.buildNodeIPEvents(ports, event) case v1.ServiceTypeClusterIP: return t.buildClusterIPEvents(event) + case v1.ServiceTypeLoadBalancer: + return t.buildLoadBalancerEvents(event) default: return events, fmt.Errorf("unsupported service type: %s", event.Service.Spec.Type) } @@ -66,6 +68,38 @@ type upstream struct { name string } +func (t *Translator) buildLoadBalancerEvents(event *core.Event) (events core.ServerUpdateEvents, err error) { + slog.Debug("Translator::buildLoadBalancerEvents", "ports", event.Service.Spec.Ports) + + addresses := make([]string, 0, len(event.Service.Status.LoadBalancer.Ingress)) + for _, ingress := range event.Service.Status.LoadBalancer.Ingress { + addresses = append(addresses, ingress.IP) + } + + for _, port := range event.Service.Spec.Ports { + context, upstreamName, err := getContextAndUpstreamName(port.Name) + if err != nil { + slog.Info("Translator::buildLoadBalancerEvents: ignoring port", "err", err, "name", port.Name) + continue + } + + upstreamServers := buildUpstreamServers(addresses, port.Port) + + switch event.Type { + case core.Created, core.Updated: + events = append(events, core.NewServerUpdateEvent(event.Type, upstreamName, context, upstreamServers)) + case core.Deleted: + events = append(events, core.NewServerUpdateEvent( + core.Updated, upstreamName, context, nil, + )) + default: + slog.Warn(`Translator::buildLoadBalancerEvents: unknown event type`, "type", event.Type) + } + } + + return events, nil +} + func (t *Translator) buildClusterIPEvents(event *core.Event) (events core.ServerUpdateEvents, err error) { namespace := event.Service.GetObjectMeta().GetNamespace() serviceName := event.Service.Name @@ -153,7 +187,7 @@ func (t *Translator) buildNodeIPEvents(ports []v1.ServicePort, event *core.Event return nil, err } - upstreamServers := buildUpstreamServers(addresses, port) + upstreamServers := buildUpstreamServers(addresses, port.NodePort) switch event.Type { case core.Created: @@ -175,11 +209,11 @@ func (t *Translator) buildNodeIPEvents(ports []v1.ServicePort, event *core.Event return events, nil } -func buildUpstreamServers(nodeIPs []string, port v1.ServicePort) core.UpstreamServers { +func buildUpstreamServers(ipAddresses []string, port int32) core.UpstreamServers { var servers core.UpstreamServers - for _, nodeIP := range nodeIPs { - host := fmt.Sprintf("%s:%d", nodeIP, port.NodePort) + for _, ip := range ipAddresses { + host := fmt.Sprintf("%s:%d", ip, port) server := core.NewUpstreamServer(host) servers = append(servers, server) } diff --git a/internal/translation/translator_test.go b/internal/translation/translator_test.go index 11fa22c..309a07c 100644 --- a/internal/translation/translator_test.go +++ b/internal/translation/translator_test.go @@ -39,8 +39,9 @@ const ( func TestCreatedTranslateNoPorts(t *testing.T) { t.Parallel() testcases := map[string]struct{ serviceType v1.ServiceType }{ - "nodePort": {v1.ServiceTypeNodePort}, - "clusterIP": {v1.ServiceTypeClusterIP}, + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + "loadBalancer": {v1.ServiceTypeLoadBalancer}, } for name, tc := range testcases { @@ -51,7 +52,7 @@ func TestCreatedTranslateNoPorts(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, 0) translator := NewTranslator( NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), @@ -74,8 +75,9 @@ func TestCreatedTranslateNoPorts(t *testing.T) { func TestCreatedTranslateNoInterestingPorts(t *testing.T) { t.Parallel() testcases := map[string]struct{ serviceType v1.ServiceType }{ - "nodePort": {v1.ServiceTypeNodePort}, - "clusterIP": {v1.ServiceTypeClusterIP}, + "nodePort": {v1.ServiceTypeNodePort}, + "clusterIP": {v1.ServiceTypeClusterIP}, + "loadBalancer": {v1.ServiceTypeLoadBalancer}, } for name, tc := range testcases { @@ -88,7 +90,7 @@ func TestCreatedTranslateNoInterestingPorts(t *testing.T) { ports := generateUpdatablePorts(portCount, 0) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, 0) translator := NewTranslator( NewFakeEndpointSliceLister([]*discovery.EndpointSlice{}, nil), @@ -114,6 +116,7 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -127,6 +130,11 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: 1, + }, } for name, tc := range testcases { @@ -139,7 +147,7 @@ func TestCreatedTranslateOneInterestingPort(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -163,6 +171,7 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -176,6 +185,11 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: 1, + }, } for name, tc := range testcases { @@ -188,7 +202,7 @@ func TestCreatedTranslateManyInterestingPorts(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -213,6 +227,7 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -226,6 +241,11 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: 1, + }, } for name, tc := range testcases { @@ -239,7 +259,7 @@ func TestCreatedTranslateManyMixedPorts(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -263,6 +283,7 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -276,6 +297,11 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), expectedServerCount: ManyEndpointSlices, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + expectedServerCount: ManyNodes, + }, } for name, tc := range testcases { @@ -288,7 +314,7 @@ func TestCreatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildCreatedEvent(service) + event := buildCreatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -316,6 +342,7 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -329,6 +356,11 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -338,7 +370,7 @@ func TestUpdatedTranslateNoPorts(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -360,6 +392,7 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -373,6 +406,11 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -384,7 +422,7 @@ func TestUpdatedTranslateNoInterestingPorts(t *testing.T) { ports := generateUpdatablePorts(portCount, 0) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -406,6 +444,7 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -419,6 +458,11 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -430,7 +474,7 @@ func TestUpdatedTranslateOneInterestingPort(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -455,6 +499,7 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -468,6 +513,11 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -479,7 +529,7 @@ func TestUpdatedTranslateManyInterestingPorts(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -504,6 +554,7 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -517,6 +568,11 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), expectedServerCount: OneEndpointSlice, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: OneNode, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -529,7 +585,7 @@ func TestUpdatedTranslateManyMixedPorts(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -554,6 +610,7 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { testcases := map[string]struct { serviceType v1.ServiceType nodes []*v1.Node + ingresses int endpoints []*discovery.EndpointSlice expectedServerCount int }{ @@ -567,6 +624,11 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { endpoints: generateEndpointSlices(ManyEndpointSlices, 6, 2), expectedServerCount: ManyEndpointSlices, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + expectedServerCount: ManyNodes, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -579,7 +641,7 @@ func TestUpdatedTranslateManyMixedPortsAndManyNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildUpdatedEvent(service) + event := buildUpdatedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -608,6 +670,7 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -615,6 +678,9 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { "clusterIP": { serviceType: v1.ServiceTypeClusterIP, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -625,7 +691,7 @@ func TestDeletedTranslateNoPortsAndNoNodes(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -650,6 +716,7 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -657,6 +724,9 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { "clusterIP": { serviceType: v1.ServiceTypeClusterIP, }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -668,7 +738,7 @@ func TestDeletedTranslateNoInterestingPortsAndNoNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, 0) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -694,6 +764,7 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -702,6 +773,9 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(0, 1, 1), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -714,7 +788,7 @@ func TestDeletedTranslateOneInterestingPortAndNoNodes(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -740,6 +814,7 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -748,6 +823,9 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(0, 4, 4), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -759,7 +837,7 @@ func TestDeletedTranslateManyInterestingPortsAndNoNodes(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -784,6 +862,7 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -792,6 +871,9 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(0, 6, 2), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + }, } for name, tc := range testcases { @@ -805,7 +887,7 @@ func TestDeletedTranslateManyMixedPortsAndNoNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -831,6 +913,7 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -840,6 +923,10 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 0, 0), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -850,7 +937,7 @@ func TestDeletedTranslateNoPortsAndOneNode(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -875,6 +962,7 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -884,6 +972,10 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 1, 0), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -895,7 +987,7 @@ func TestDeletedTranslateNoInterestingPortsAndOneNode(t *testing.T) { ports := generateUpdatablePorts(portCount, 0) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -921,6 +1013,7 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -930,6 +1023,10 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 1, 1), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -942,7 +1039,7 @@ func TestDeletedTranslateOneInterestingPortAndOneNode(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -968,6 +1065,7 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -977,6 +1075,10 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 4, 4), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -989,7 +1091,7 @@ func TestDeletedTranslateManyInterestingPortsAndOneNode(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1014,6 +1116,7 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1023,6 +1126,10 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(OneEndpointSlice, 6, 2), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: OneNode, + }, } for name, tc := range testcases { @@ -1035,7 +1142,7 @@ func TestDeletedTranslateManyMixedPortsAndOneNode(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1061,6 +1168,7 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1070,6 +1178,10 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(ManyEndpointSlices, 0, 0), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -1079,7 +1191,7 @@ func TestDeletedTranslateNoPortsAndManyNodes(t *testing.T) { const expectedEventCount = 0 service := defaultService(tc.serviceType) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1104,6 +1216,7 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1113,6 +1226,10 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 0), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -1125,7 +1242,7 @@ func TestDeletedTranslateNoInterestingPortsAndManyNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1151,6 +1268,7 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1160,6 +1278,10 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(ManyEndpointSlices, 1, 1), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -1171,7 +1293,7 @@ func TestDeletedTranslateOneInterestingPortAndManyNodes(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1197,6 +1319,7 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1206,6 +1329,10 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { serviceType: v1.ServiceTypeClusterIP, endpoints: generateEndpointSlices(ManyEndpointSlices, 4, 4), }, + "loadBalancer": { + serviceType: v1.ServiceTypeLoadBalancer, + ingresses: ManyNodes, + }, } for name, tc := range testcases { @@ -1217,7 +1344,7 @@ func TestDeletedTranslateManyInterestingPortsAndManyNodes(t *testing.T) { ports := generatePorts(portCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1242,6 +1369,7 @@ func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { serviceType v1.ServiceType nodes []*v1.Node endpoints []*discovery.EndpointSlice + ingresses int }{ "nodePort": { serviceType: v1.ServiceTypeNodePort, @@ -1263,7 +1391,7 @@ func TestDeletedTranslateManyMixedPortsAndManyNodes(t *testing.T) { ports := generateUpdatablePorts(portCount, updatablePortCount) service := serviceWithPorts(tc.serviceType, ports) - event := buildDeletedEvent(service) + event := buildDeletedEvent(service, tc.ingresses) translator := NewTranslator(NewFakeEndpointSliceLister(tc.endpoints, nil), NewFakeNodeLister(tc.nodes, nil)) translatedEvents, err := translator.Translate(&event) @@ -1311,21 +1439,27 @@ func serviceWithPorts(serviceType v1.ServiceType, ports []v1.ServicePort) *v1.Se } } -func buildCreatedEvent(service *v1.Service) core.Event { - return buildEvent(core.Created, service) +func buildCreatedEvent(service *v1.Service, ingressCount int) core.Event { + return buildEvent(core.Created, service, ingressCount) } -func buildDeletedEvent(service *v1.Service) core.Event { - return buildEvent(core.Deleted, service) +func buildDeletedEvent(service *v1.Service, ingressCount int) core.Event { + return buildEvent(core.Deleted, service, ingressCount) } -func buildUpdatedEvent(service *v1.Service) core.Event { - return buildEvent(core.Updated, service) +func buildUpdatedEvent(service *v1.Service, ingressCount int) core.Event { + return buildEvent(core.Updated, service, ingressCount) } -func buildEvent(eventType core.EventType, service *v1.Service) core.Event { +func buildEvent(eventType core.EventType, service *v1.Service, ingressCount int) core.Event { event := core.NewEvent(eventType, service) event.Service.Name = "default-service" + ingresses := make([]v1.LoadBalancerIngress, 0, ingressCount) + for i := range ingressCount { + ingress := v1.LoadBalancerIngress{IP: fmt.Sprintf("ipAddress%d", i)} + ingresses = append(ingresses, ingress) + } + event.Service.Status.LoadBalancer.Ingress = ingresses return event } From 82c81a78b5d7541c01de4e6ea788c3eb56b546a6 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 10 Feb 2025 14:46:31 -0700 Subject: [PATCH 111/136] NLB-6320 Bumped to version 1.1.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 3eefcb9..9084fa2 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.0.0 +1.1.0 From 6e48c5de3b04fa9cdc9c2f6d70190ca844bb0170 Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 27 Feb 2025 12:11:03 -0800 Subject: [PATCH 112/136] Remove AZ_LOCATION for stacks We should not be setting this value and instead rely on the tooling defaults. --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 04dffff..7f73a13 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -216,7 +216,6 @@ trigger-e2e: TEST_NLK_CHART_URL: "oci://${DEVOPS_DOCKER_URL_DEFAULT}/nginx-azure-lb/${CI_PROJECT_NAME}/charts/${CI_COMMIT_REF_SLUG}/nginxaas-loadbalancer-kubernetes" TEST_NLK_IMG_TAG: ${CI_COMMIT_SHORT_SHA} TEST_ARGS: "test_nlk.py" - AZ_LOCATION: "West Central US" trigger: project: f5/nginx/nginxazurelb/tools/nlbtest branch: main From 34c3f7cf92bfc3973435ef60e240397c8fc793ba Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 11:11:04 -0700 Subject: [PATCH 113/136] NLB-6293 Updated k8s libraries used by NLK This will help fix vulnerabilities discoved by mend in k8s.io/apiMAChinery-v0.26.0. --- go.mod | 58 +++++++-------- go.sum | 229 ++++++++++++++++++--------------------------------------- 2 files changed, 100 insertions(+), 187 deletions(-) diff --git a/go.mod b/go.mod index f7c3dc8..5943bf6 100644 --- a/go.mod +++ b/go.mod @@ -4,40 +4,38 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.22.6 - -toolchain go1.23.4 +go 1.23.0 require ( github.com/nginx/nginx-plus-go-client/v2 v2.3.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.10.0 - k8s.io/api v0.26.0 - k8s.io/apimachinery v0.26.0 - k8s.io/client-go v0.26.0 + golang.org/x/sync v0.11.0 + k8s.io/api v0.32.2 + k8s.io/apimachinery v0.32.2 + k8s.io/client-go v0.32.2 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -52,27 +50,27 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/x448/float16 v0.8.4 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.33.0 // indirect - golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.5.0 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/time v0.7.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index 505b7f6..e03c187 100644 --- a/go.sum +++ b/go.sum @@ -1,76 +1,53 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -79,10 +56,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -94,12 +69,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nginx/nginx-plus-go-client/v2 v2.3.0 h1:ciKh1lwadNzUaOGjLcKWu/BGigASxU6p7v/6US71fhA= github.com/nginx/nginx-plus-go-client/v2 v2.3.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= -github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -107,9 +80,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -124,24 +96,23 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= @@ -149,131 +120,75 @@ go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTV golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= +golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= -k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= -k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= -k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= -k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= +k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= +k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= +k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= +k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= From da3ab085eb7e1d80b7072c80ec34a24b6bd5277d Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 11:12:09 -0700 Subject: [PATCH 114/136] NLB-6293 Added mutex lock around certification's Certificates type This enables concurrency-safe access by multiple goroutines. --- cmd/certificates-test-harness/main.go | 4 +- cmd/tls-config-factory-test-harness/main.go | 40 +++--- internal/authentication/factory_test.go | 130 ++++++++++---------- internal/certification/certificates.go | 61 +++++---- internal/certification/certificates_test.go | 15 ++- 5 files changed, 136 insertions(+), 114 deletions(-) diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go index f3468a9..898c3a3 100644 --- a/cmd/certificates-test-harness/main.go +++ b/cmd/certificates-test-harness/main.go @@ -37,14 +37,14 @@ func run() error { return fmt.Errorf(`error building a Kubernetes client: %w`, err) } - certificates := certification.NewCertificates(ctx, k8sClient) + certificates := certification.NewCertificates(k8sClient, nil) err = certificates.Initialize() if err != nil { return fmt.Errorf(`error occurred initializing certificates: %w`, err) } - go certificates.Run() //nolint:errcheck + go certificates.Run(ctx) //nolint:errcheck <-ctx.Done() return nil diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go index 7883d65..638b0bc 100644 --- a/cmd/tls-config-factory-test-harness/main.go +++ b/cmd/tls-config-factory-test-harness/main.go @@ -89,28 +89,28 @@ func buildConfigMap() map[string]TLSConfiguration { } func ssTLSConfig() configuration.Settings { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) return configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.SelfSignedTLS, + Certificates: certificates, } } func ssMtlsConfig() configuration.Settings { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) return configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.SelfSignedMutualTLS, + Certificates: certificates, } } @@ -121,14 +121,14 @@ func caTLSConfig() configuration.Settings { } func caMtlsConfig() configuration.Settings { - certificates := make(map[string]map[string]core.SecretBytes) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) return configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.CertificateAuthorityMutualTLS, + Certificates: certificates, } } diff --git a/internal/authentication/factory_test.go b/internal/authentication/factory_test.go index 6b5fcaf..4777607 100644 --- a/internal/authentication/factory_test.go +++ b/internal/authentication/factory_test.go @@ -37,16 +37,16 @@ func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.SelfSignedTLS, + Certificates: certificates, } tlsConfig, err := NewTLSConfig(settings) @@ -73,14 +73,16 @@ func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.SelfSignedTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) @@ -95,16 +97,16 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificateDataPEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificateDataPEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.SelfSignedTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) @@ -119,17 +121,17 @@ func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.SelfSignedMutualTLS, + Certificates: certificates, } tlsConfig, err := NewTLSConfig(settings) @@ -156,15 +158,17 @@ func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.SelfSignedMutualTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) @@ -179,17 +183,17 @@ func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.SelfSignedMutualTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) @@ -232,16 +236,16 @@ func TestTlsFactory_CaTlsMode(t *testing.T) { func TestTlsFactory_CaMtlsMode(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - CaCertificateSecretKey: CaCertificateSecretKey, - ClientCertificateSecretKey: ClientCertificateSecretKey, - }, + TLSMode: configuration.CertificateAuthorityMutualTLS, + Certificates: certificates, } tlsConfig, err := NewTLSConfig(settings) @@ -268,15 +272,17 @@ func TestTlsFactory_CaMtlsMode(t *testing.T) { func TestTlsFactory_CaMtlsModeClientCertificateError(t *testing.T) { t.Parallel() - certificates := make(map[string]map[string]core.SecretBytes) - certificates[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certificates[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) + certs := make(map[string]map[string]core.SecretBytes) + certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) + certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) + + certificates := certification.NewCertificates(nil, certs) + certificates.CaCertificateSecretKey = CaCertificateSecretKey + certificates.ClientCertificateSecretKey = ClientCertificateSecretKey settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: &certification.Certificates{ - Certificates: certificates, - }, + TLSMode: configuration.CertificateAuthorityMutualTLS, + Certificates: certificates, } _, err := NewTLSConfig(settings) diff --git a/internal/certification/certificates.go b/internal/certification/certificates.go index c59eb0e..86b9320 100644 --- a/internal/certification/certificates.go +++ b/internal/certification/certificates.go @@ -13,6 +13,7 @@ import ( "context" "fmt" "log/slog" + "sync" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" @@ -34,10 +35,8 @@ const ( ) type Certificates struct { - Certificates map[string]map[string]core.SecretBytes - - // Context is the context used to control the application. - Context context.Context + mu sync.Mutex // guards Certificates + certificates map[string]map[string]core.SecretBytes // CaCertificateSecretKey is the name of the Secret that contains the Certificate Authority certificate. CaCertificateSecretKey string @@ -56,25 +55,32 @@ type Certificates struct { } // NewCertificates factory method that returns a new Certificates object. -func NewCertificates(ctx context.Context, k8sClient kubernetes.Interface) *Certificates { +func NewCertificates( + k8sClient kubernetes.Interface, certificates map[string]map[string]core.SecretBytes, +) *Certificates { return &Certificates{ k8sClient: k8sClient, - Context: ctx, - Certificates: nil, + certificates: certificates, } } // GetCACertificate returns the Certificate Authority certificate. func (c *Certificates) GetCACertificate() core.SecretBytes { - bytes := c.Certificates[c.CaCertificateSecretKey][CertificateKey] + c.mu.Lock() + defer c.mu.Unlock() + + bytes := c.certificates[c.CaCertificateSecretKey][CertificateKey] return bytes } // GetClientCertificate returns the Client certificate and key. func (c *Certificates) GetClientCertificate() (core.SecretBytes, core.SecretBytes) { - keyBytes := c.Certificates[c.ClientCertificateSecretKey][CertificateKeyKey] - certificateBytes := c.Certificates[c.ClientCertificateSecretKey][CertificateKey] + c.mu.Lock() + defer c.mu.Unlock() + + keyBytes := c.certificates[c.ClientCertificateSecretKey][CertificateKeyKey] + certificateBytes := c.certificates[c.ClientCertificateSecretKey][CertificateKey] return keyBytes, certificateBytes } @@ -85,7 +91,9 @@ func (c *Certificates) Initialize() error { var err error - c.Certificates = make(map[string]map[string]core.SecretBytes) + c.mu.Lock() + c.certificates = make(map[string]map[string]core.SecretBytes) + c.mu.Unlock() informer := c.buildInformer() @@ -100,16 +108,16 @@ func (c *Certificates) Initialize() error { } // Run starts the SharedInformer. -func (c *Certificates) Run() error { +func (c *Certificates) Run(ctx context.Context) error { slog.Info("Certificates::Run") if c.informer == nil { return fmt.Errorf(`initialize must be called before Run`) } - c.informer.Run(c.Context.Done()) + c.informer.Run(ctx.Done()) - <-c.Context.Done() + <-ctx.Done() return nil } @@ -152,17 +160,20 @@ func (c *Certificates) handleAddEvent(obj interface{}) { return } - c.Certificates[secret.Name] = map[string]core.SecretBytes{} + c.mu.Lock() + defer c.mu.Unlock() + + c.certificates[secret.Name] = map[string]core.SecretBytes{} // Input from the secret comes in the form // tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVCVEN... // tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0l... // Where the keys are `tls.crt` and `tls.key` and the values are []byte for k, v := range secret.Data { - c.Certificates[secret.Name][k] = core.SecretBytes(v) + c.certificates[secret.Name][k] = core.SecretBytes(v) } - slog.Debug("Certificates::handleAddEvent", slog.Int("certCount", len(c.Certificates))) + slog.Debug("Certificates::handleAddEvent", slog.Int("certCount", len(c.certificates))) } func (c *Certificates) handleDeleteEvent(obj interface{}) { @@ -174,11 +185,14 @@ func (c *Certificates) handleDeleteEvent(obj interface{}) { return } - if c.Certificates[secret.Name] != nil { - delete(c.Certificates, secret.Name) + c.mu.Lock() + defer c.mu.Unlock() + + if c.certificates[secret.Name] != nil { + delete(c.certificates, secret.Name) } - slog.Debug("Certificates::handleDeleteEvent", slog.Int("certCount", len(c.Certificates))) + slog.Debug("Certificates::handleDeleteEvent", slog.Int("certCount", len(c.certificates))) } func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { @@ -190,9 +204,12 @@ func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { return } + c.mu.Lock() + defer c.mu.Unlock() + for k, v := range secret.Data { - c.Certificates[secret.Name][k] = v + c.certificates[secret.Name][k] = v } - slog.Debug("Certificates::handleUpdateEvent", slog.Int("certCount", len(c.Certificates))) + slog.Debug("Certificates::handleUpdateEvent", slog.Int("certCount", len(c.certificates))) } diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go index 901964a..7d2363b 100644 --- a/internal/certification/certificates_test.go +++ b/internal/certification/certificates_test.go @@ -23,9 +23,8 @@ const ( func TestNewCertificate(t *testing.T) { t.Parallel() - ctx := context.Background() - certificates := NewCertificates(ctx, nil) + certificates := NewCertificates(nil, nil) if certificates == nil { t.Fatalf(`certificates should not be nil`) @@ -34,7 +33,7 @@ func TestNewCertificate(t *testing.T) { func TestCertificates_Initialize(t *testing.T) { t.Parallel() - certificates := NewCertificates(context.Background(), nil) + certificates := NewCertificates(nil, nil) err := certificates.Initialize() if err != nil { @@ -44,9 +43,9 @@ func TestCertificates_Initialize(t *testing.T) { func TestCertificates_RunWithoutInitialize(t *testing.T) { t.Parallel() - certificates := NewCertificates(context.Background(), nil) + certificates := NewCertificates(nil, nil) - err := certificates.Run() + err := certificates.Run(context.Background()) if err == nil { t.Fatalf(`Expected error`) } @@ -58,7 +57,7 @@ func TestCertificates_RunWithoutInitialize(t *testing.T) { func TestCertificates_EmptyCertificates(t *testing.T) { t.Parallel() - certificates := NewCertificates(context.Background(), nil) + certificates := NewCertificates(nil, nil) err := certificates.Initialize() if err != nil { @@ -86,7 +85,7 @@ func TestCertificates_ExerciseHandlers(t *testing.T) { k8sClient := fake.NewSimpleClientset() - certificates := NewCertificates(ctx, k8sClient) + certificates := NewCertificates(k8sClient, nil) _ = certificates.Initialize() @@ -94,7 +93,7 @@ func TestCertificates_ExerciseHandlers(t *testing.T) { //nolint:govet,staticcheck go func() { - err := certificates.Run() + err := certificates.Run(context.Background()) assert.NoError(t, err, "expected no error running certificates") }() From 8b688a315d6c46b668a172b1be5a571901b8dbfe Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 11:20:49 -0700 Subject: [PATCH 115/136] NLB-6293 Upgraded golangci-lint to v1.64.5 and fixed configuration --- .golangci.yml | 18 +++++++----------- Makefile | 2 +- cmd/nginx-loadbalancer-kubernetes/main.go | 4 ++++ internal/synchronization/synchronizer.go | 3 +++ 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 6cb3587..6f0fddf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -26,7 +26,6 @@ linters: - misspell - nakedret - prealloc - - exportloopref - stylecheck - unconvert - unparam @@ -38,20 +37,17 @@ linters: run: # 10 minute timeout for analysis timeout: 10m - skip-dirs-use-default: true - skip-dirs: - - .go/pkg/mod - - pkg/spec/api # Generated code - - vendor - - vendor-fork # Specific linter settings linters-settings: gocyclo: # Minimal code complexity to report min-complexity: 16 govet: - # Report shadowed variables - check-shadowing: true + disable-all: true + enable: + # Report shadowed variables + - shadow + misspell: # Correct spellings using locale preferences for US locale: US @@ -63,8 +59,8 @@ linters-settings: # If this list is empty, all structs are tested. # Default: [] include: - - 'gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/pkg/token.TokenID' - - 'gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/internal/dpo/agent/certificates.CertGetRequest' + - "gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/pkg/token.TokenID" + - "gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/internal/dpo/agent/certificates.CertGetRequest" issues: # Exclude configuration diff --git a/Makefile b/Makefile index 4d00656..354bd95 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ tools: @go install gotest.tools/gotestsum@latest @go install golang.org/x/tools/cmd/goimports@latest @go install github.com/jstemmer/go-junit-report@v1.0.0 - @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.57.1 + @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin v1.64.5 deps: @go mod download diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index 5aba57f..d2d923c 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -133,9 +133,13 @@ func buildKubernetesClient() (*kubernetes.Clientset, error) { return client, nil } +// TODO: NLB-6294 change to use new typed workqueues +// +//nolint:staticcheck //ignore deprecation warnings func buildWorkQueue(settings configuration.WorkQueueSettings) workqueue.RateLimitingInterface { slog.Debug("Watcher::buildSynchronizerWorkQueue") + //nolint:staticcheck //ignore deprecation warnings rateLimiter := workqueue.NewItemExponentialFailureRateLimiter(settings.RateLimiterBase, settings.RateLimiterMax) return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 82b3071..3810805 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -50,6 +50,8 @@ type ServiceKey struct { // a Border Client as specified in the Service annotation for the Upstream. // See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { + // TODO: NLB-6294 change to use new typed workqueues + //nolint:staticcheck //ignore deprecation warnings eventQueue workqueue.RateLimitingInterface settings configuration.Settings translator Translator @@ -60,6 +62,7 @@ type Synchronizer struct { // NewSynchronizer creates a new Synchronizer. func NewSynchronizer( settings configuration.Settings, + //nolint:staticcheck //ignore deprecation warnings eventQueue workqueue.RateLimitingInterface, translator Translator, serviceLister corelisters.ServiceLister, From d02645113da736a9d2c7941d8bce6891da26423a Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 15:19:01 -0700 Subject: [PATCH 116/136] NLB-6294 The synchronizer uses a typed rate-limited workqueue The latest versions of the kubernetes libraries recommend using a typed workqueue and this reduces a bit of boilerplate and error handling, because we no longer have to cast the workitems returned by the queue into the desired types. --- cmd/nginx-loadbalancer-kubernetes/main.go | 13 ++++----- internal/synchronization/synchronizer.go | 15 ++-------- internal/synchronization/synchronizer_test.go | 14 ++++----- test/mocks/mock_ratelimitinginterface.go | 29 +++++++++---------- 4 files changed, 30 insertions(+), 41 deletions(-) diff --git a/cmd/nginx-loadbalancer-kubernetes/main.go b/cmd/nginx-loadbalancer-kubernetes/main.go index d2d923c..43de794 100644 --- a/cmd/nginx-loadbalancer-kubernetes/main.go +++ b/cmd/nginx-loadbalancer-kubernetes/main.go @@ -133,13 +133,12 @@ func buildKubernetesClient() (*kubernetes.Clientset, error) { return client, nil } -// TODO: NLB-6294 change to use new typed workqueues -// -//nolint:staticcheck //ignore deprecation warnings -func buildWorkQueue(settings configuration.WorkQueueSettings) workqueue.RateLimitingInterface { +func buildWorkQueue(settings configuration.WorkQueueSettings, +) workqueue.TypedRateLimitingInterface[synchronization.ServiceKey] { slog.Debug("Watcher::buildSynchronizerWorkQueue") - //nolint:staticcheck //ignore deprecation warnings - rateLimiter := workqueue.NewItemExponentialFailureRateLimiter(settings.RateLimiterBase, settings.RateLimiterMax) - return workqueue.NewNamedRateLimitingQueue(rateLimiter, settings.Name) + rateLimiter := workqueue.NewTypedItemExponentialFailureRateLimiter[synchronization.ServiceKey]( + settings.RateLimiterBase, settings.RateLimiterMax) + return workqueue.NewTypedRateLimitingQueueWithConfig( + rateLimiter, workqueue.TypedRateLimitingQueueConfig[synchronization.ServiceKey]{Name: settings.Name}) } diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 3810805..50717d7 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -50,9 +50,7 @@ type ServiceKey struct { // a Border Client as specified in the Service annotation for the Upstream. // See application/border_client.go and application/application_constants.go for details. type Synchronizer struct { - // TODO: NLB-6294 change to use new typed workqueues - //nolint:staticcheck //ignore deprecation warnings - eventQueue workqueue.RateLimitingInterface + eventQueue workqueue.TypedRateLimitingInterface[ServiceKey] settings configuration.Settings translator Translator cache *cache @@ -62,8 +60,7 @@ type Synchronizer struct { // NewSynchronizer creates a new Synchronizer. func NewSynchronizer( settings configuration.Settings, - //nolint:staticcheck //ignore deprecation warnings - eventQueue workqueue.RateLimitingInterface, + eventQueue workqueue.TypedRateLimitingInterface[ServiceKey], translator Translator, serviceLister corelisters.ServiceLister, ) (*Synchronizer, error) { @@ -292,13 +289,7 @@ func (s *Synchronizer) handleNextServiceEvent(ctx context.Context) bool { defer s.eventQueue.Done(svc) - key, ok := svc.(ServiceKey) - if !ok { - slog.Warn(`Synchronizer::handleNextServiceEvent: invalid service type`, "service", svc) - return true - } - - s.withRetry(s.handleServiceEvent(ctx, key), key) + s.withRetry(s.handleServiceEvent(ctx, svc), svc) return true } diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index 7592f5c..c49c718 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -22,7 +22,7 @@ import ( func TestSynchronizer_NewSynchronizer(t *testing.T) { t.Parallel() - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( configuration.Settings{}, @@ -43,7 +43,7 @@ func TestSynchronizer_AddEventNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings(), @@ -73,7 +73,7 @@ func TestSynchronizer_AddEventOneHost(t *testing.T) { const expectedEventCount = 1 events := buildServerUpdateEvents(1) - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings("https://localhost:8080"), @@ -106,7 +106,7 @@ func TestSynchronizer_AddEventManyHosts(t *testing.T) { "https://localhost:8082", } - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings(hosts...), @@ -133,7 +133,7 @@ func TestSynchronizer_AddEventsNoHosts(t *testing.T) { t.Parallel() const expectedEventCount = 0 events := buildServerUpdateEvents(4) - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings(), @@ -165,7 +165,7 @@ func TestSynchronizer_AddEventsOneHost(t *testing.T) { t.Parallel() const expectedEventCount = 4 events := buildServerUpdateEvents(1) - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} synchronizer, err := NewSynchronizer( defaultSettings("https://localhost:8080"), @@ -195,7 +195,7 @@ func TestSynchronizer_AddEventsManyHosts(t *testing.T) { t.Parallel() const eventCount = 4 events := buildServerUpdateEvents(eventCount) - rateLimiter := &mocks.MockRateLimiter{} + rateLimiter := &mocks.MockRateLimiter[ServiceKey]{} hosts := []string{ "https://localhost:8080", diff --git a/test/mocks/mock_ratelimitinginterface.go b/test/mocks/mock_ratelimitinginterface.go index ee3ccd4..d5da3b7 100644 --- a/test/mocks/mock_ratelimitinginterface.go +++ b/test/mocks/mock_ratelimitinginterface.go @@ -7,51 +7,50 @@ package mocks import "time" -type MockRateLimiter struct { - items []interface{} +type MockRateLimiter[T any] struct { + items []T } -func (m *MockRateLimiter) Add(_ interface{}) { +func (m *MockRateLimiter[T]) Add(_ T) { } -func (m *MockRateLimiter) Len() int { +func (m *MockRateLimiter[T]) Len() int { return len(m.items) } -func (m *MockRateLimiter) Get() (item interface{}, shutdown bool) { +func (m *MockRateLimiter[T]) Get() (item T, shutdown bool) { if len(m.items) > 0 { item = m.items[0] m.items = m.items[1:] return item, false } - return nil, false + return item, false } -func (m *MockRateLimiter) Done(_ interface{}) { +func (m *MockRateLimiter[T]) Done(_ T) { } -func (m *MockRateLimiter) ShutDown() { +func (m *MockRateLimiter[T]) ShutDown() { } -func (m *MockRateLimiter) ShutDownWithDrain() { +func (m *MockRateLimiter[T]) ShutDownWithDrain() { } -func (m *MockRateLimiter) ShuttingDown() bool { +func (m *MockRateLimiter[T]) ShuttingDown() bool { return true } -func (m *MockRateLimiter) AddAfter(item interface{}, _ time.Duration) { +func (m *MockRateLimiter[T]) AddAfter(item T, _ time.Duration) { m.items = append(m.items, item) } -func (m *MockRateLimiter) AddRateLimited(item interface{}) { +func (m *MockRateLimiter[T]) AddRateLimited(item T) { m.items = append(m.items, item) } -func (m *MockRateLimiter) Forget(_ interface{}) { - +func (m *MockRateLimiter[T]) Forget(_ T) { } -func (m *MockRateLimiter) NumRequeues(_ interface{}) int { +func (m *MockRateLimiter[T]) NumRequeues(_ T) int { return 0 } From a2c35961f92f5aefc440e46876fcec34f7bec2e0 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 27 Feb 2025 15:21:53 -0700 Subject: [PATCH 117/136] NLB-6294 Bumped version to 1.1.1 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 9084fa2..524cb55 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.0 +1.1.1 From 88cefab520d0c518376e612fad9bc791f77a2331 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 28 Feb 2025 09:07:24 -0700 Subject: [PATCH 118/136] NLB-6294 NewTransport clones the default http.DefaultTransport variable Multiple parallel tests were all accessing the same pointer to a single variable for the DefaultTransport in the http package. This was leading to data races in unit tests. --- internal/communication/factory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/communication/factory.go b/internal/communication/factory.go index eec593b..cf1bfcb 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -63,7 +63,7 @@ func NewTLSConfig(settings configuration.Settings) *tls.Config { // NewTransport is a factory method to create a new basic Http Transport. func NewTransport(config *tls.Config) *netHttp.Transport { - transport := netHttp.DefaultTransport.(*netHttp.Transport) + transport := netHttp.DefaultTransport.(*netHttp.Transport).Clone() transport.TLSClientConfig = config return transport From 9e3dde695fe128a90c19e7ed460278f84d06ac7d Mon Sep 17 00:00:00 2001 From: sarna Date: Thu, 13 Mar 2025 10:55:53 -0700 Subject: [PATCH 119/136] Force linux builds for the product We don't really have a need to handle multi arch images yet and all our local workflows are tied to using AKS. While that might change in the future, for the time being, we want to make sure to build and publish linux/amd64 images to match the test envs that we have. --- scripts/docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/docker.sh b/scripts/docker.sh index 6208b79..cdfcd82 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -13,6 +13,7 @@ build() { --tag "${repo}:${CI_COMMIT_REF_SLUG}" \ --tag "${repo}:${CI_COMMIT_REF_SLUG}-$version" \ --tag "${repo}:${CI_COMMIT_SHORT_SHA}" \ + --platform "linux/amd64" \ -f "${ROOT_DIR}/Dockerfile" . } From 0ae1eb7de6ab55a78d0453f0071f94b88a20d8af Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 15 Apr 2025 13:40:07 -0600 Subject: [PATCH 120/136] Added documents on how to release NLK --- RELEASE.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 RELEASE.md diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..7b18e6e --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,50 @@ +# Releasing NLK + +## Releasing a version to our internal dockerhub registry + +- Go to the [NLK repo](https://gitlab.com/f5/nginx/nginxazurelb/nginxaas-loadbalancer-kubernetes). + +- [Create a new tag](https://gitlab.com/f5/nginx/nginxazurelb/nginxaas-loadbalancer-kubernetes/-/tags/new). + +- Give the tag the name of the version to be released, e.g. "v1.1.1". This should match the version in the `version` file at the root of the repo. + +- Under **Create from** select the branch from which the image will be created. + +- Hit **Create tag**. + +## Releasing a version to Azure Marketplace + +This workflow requires Azure Marketplace permissions which few members of NGINXaaS possess (currently Ken, Ashok and Ryan). + +- Complete the steps above to publish the image internally. + +- Go to "Marketplace Offers" + +- Click on "F5 NGINX LoadBalancer for Kubernetes" + +- Under "Plan overview" select the plan which has a "Live" status (this should be "F5 NGINXaaS AKS Extension") + +- On a panel on the left hand side, select "Technical Configuration" + +- A pop up appears. + - Under "Registry" select the "nlbmarketplaceacrprod" option + - Under "Repo" select "marketplace/nginxaas-loadbalancer-kubernetes" + - Under "CNAB Bundle" select the version you wish to publish + +- To complete the publishing of the image click "Add CNAB Button" button on the bottom of the popup. + +- Select "Save draft". + +- This should take you to a "Review and Publish" screen. If the UI seems to stall. Follow steps below. + + - Next to "Marketplace offers" on the top of the screen, select "F5 NGINX Loadbalancer for Kubernetes". + + - Select "Offer overview" from the panel on the left. + + - Next to the heading "F5 NGINX Loadbalancer for Kubernetes | Offer Overview" select "Review and publish" + +- A number of items should appear, but they must include "F5 NGINXaaS AKS extenstion". Leave all items as they are. + +- Then click "Publish". + +- This will take a while. Check in on it after 24 hours. From 97cd5fb546b793adb7fa2b7c7e71a735410c3321 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 13 Mar 2025 16:06:20 -0600 Subject: [PATCH 121/136] Removed synhronization's random functions These functions were being used to create IDs that were not really necessary for the business logic and which were generating security alerts because of weak cryptography techniques. --- internal/core/server_update_event.go | 9 +---- internal/core/server_update_event_test.go | 10 +---- internal/synchronization/rand.go | 39 ------------------- internal/synchronization/synchronizer.go | 11 +++--- internal/synchronization/synchronizer_test.go | 1 - 5 files changed, 8 insertions(+), 62 deletions(-) delete mode 100644 internal/synchronization/rand.go diff --git a/internal/core/server_update_event.go b/internal/core/server_update_event.go index 0bc8868..04d2dc0 100644 --- a/internal/core/server_update_event.go +++ b/internal/core/server_update_event.go @@ -9,14 +9,10 @@ package core // from Events received from the Handler. These are then consumed by the Synchronizer and passed along to // the appropriate BorderClient. type ServerUpdateEvent struct { - // ClientType is the type of BorderClient that should handle this event. This is configured via Service Annotations. // See application_constants.go for the list of supported types. ClientType string - // Id is the unique identifier for this event. - ID string - // NginxHost is the host name of the NGINX Plus instance that should handle this event. NginxHost string @@ -48,11 +44,10 @@ func NewServerUpdateEvent( } } -// ServerUpdateEventWithIDAndHost creates a new ServerUpdateEvent with the specified Id and Host. -func ServerUpdateEventWithIDAndHost(event *ServerUpdateEvent, id string, nginxHost string) *ServerUpdateEvent { +// ServerUpdateEventWithHost creates a new ServerUpdateEvent with the specified Host. +func ServerUpdateEventWithHost(event *ServerUpdateEvent, nginxHost string) *ServerUpdateEvent { return &ServerUpdateEvent{ ClientType: event.ClientType, - ID: id, NginxHost: nginxHost, Type: event.Type, UpstreamName: event.UpstreamName, diff --git a/internal/core/server_update_event_test.go b/internal/core/server_update_event_test.go index 9f36002..3be4702 100644 --- a/internal/core/server_update_event_test.go +++ b/internal/core/server_update_event_test.go @@ -17,19 +17,11 @@ func TestServerUpdateEventWithIdAndHost(t *testing.T) { t.Parallel() event := NewServerUpdateEvent(Created, "upstream", clientType, emptyUpstreamServers) - if event.ID != "" { - t.Errorf("expected empty ID, got %s", event.ID) - } - if event.NginxHost != "" { t.Errorf("expected empty NginxHost, got %s", event.NginxHost) } - eventWithIDAndHost := ServerUpdateEventWithIDAndHost(event, "id", "host") - - if eventWithIDAndHost.ID != "id" { - t.Errorf("expected Id to be 'id', got %s", eventWithIDAndHost.ID) - } + eventWithIDAndHost := ServerUpdateEventWithHost(event, "host") if eventWithIDAndHost.NginxHost != "host" { t.Errorf("expected NginxHost to be 'host', got %s", eventWithIDAndHost.NginxHost) diff --git a/internal/synchronization/rand.go b/internal/synchronization/rand.go deleted file mode 100644 index 6bf58d1..0000000 --- a/internal/synchronization/rand.go +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package synchronization - -import ( - // Try using crpyto if needed. - "math/rand" - "time" -) - -// charset contains all characters that can be used in random string generation -var charset = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -// number contains all numbers that can be used in random string generation -var number = []byte("0123456789") - -// alphaNumeric contains all characters and numbers that can be used in random string generation -var alphaNumeric = append(charset, number...) - -// RandomString where n is the length of random string we want to generate -func RandomString(n int) string { - b := make([]byte, n) - for i := range b { - // randomly select 1 character from given charset - b[i] = alphaNumeric[rand.Intn(len(alphaNumeric))] //nolint:gosec - } - return string(b) -} - -// RandomMilliseconds returns a random duration between min and max milliseconds -func RandomMilliseconds(min, max int) time.Duration { - randomizer := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec - random := randomizer.Intn(max-min) + min - - return time.Millisecond * time.Duration(random) -} diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 50717d7..80faaa3 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -146,10 +146,9 @@ func (s *Synchronizer) fanOutEventToHosts(event core.ServerUpdateEvents) core.Se var events core.ServerUpdateEvents - for hidx, host := range s.settings.NginxPlusHosts { - for eidx, event := range event { - id := fmt.Sprintf(`[%d:%d]-[%s]-[%s]-[%s]`, hidx, eidx, RandomString(12), event.UpstreamName, host) - updatedEvent := core.ServerUpdateEventWithIDAndHost(event, id, host) + for _, host := range s.settings.NginxPlusHosts { + for _, event := range event { + updatedEvent := core.ServerUpdateEventWithHost(event, host) events = append(events, updatedEvent) } @@ -237,7 +236,7 @@ func (s *Synchronizer) handleServiceEvent(ctx context.Context, key ServiceKey) ( // handleCreatedUpdatedEvent handles events of type Created or Updated. func (s *Synchronizer) handleCreatedUpdatedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { - slog.Debug(`Synchronizer::handleCreatedUpdatedEvent`, "eventID", serverUpdateEvent.ID) + slog.Debug(`Synchronizer::handleCreatedUpdatedEvent`) var err error @@ -255,7 +254,7 @@ func (s *Synchronizer) handleCreatedUpdatedEvent(ctx context.Context, serverUpda // handleDeletedEvent handles events of type Deleted. func (s *Synchronizer) handleDeletedEvent(ctx context.Context, serverUpdateEvent *core.ServerUpdateEvent) error { - slog.Debug(`Synchronizer::handleDeletedEvent`, "eventID", serverUpdateEvent.ID) + slog.Debug(`Synchronizer::handleDeletedEvent`) var err error diff --git a/internal/synchronization/synchronizer_test.go b/internal/synchronization/synchronizer_test.go index c49c718..ba2253b 100644 --- a/internal/synchronization/synchronizer_test.go +++ b/internal/synchronization/synchronizer_test.go @@ -244,7 +244,6 @@ func buildServerUpdateEvents(count int) core.ServerUpdateEvents { events := make(core.ServerUpdateEvents, count) for i := 0; i < count; i++ { events[i] = &core.ServerUpdateEvent{ - ID: fmt.Sprintf("id-%v", i), NginxHost: "https://localhost:8080", Type: 0, UpstreamName: "", From 927ed461e82d73ee5653fa6a71f8d9cc7ed1dfb8 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 21 Mar 2025 14:46:22 -0600 Subject: [PATCH 122/136] NGINX plus http client rejects requests with too many headers The http client is processing requests created by the nginx plus client library, and that library should always include a sensible number of headers. But the lack of change on the number of headers was causing security vulnerability flags to be raised over denial of service resource exhaustion attacks. --- internal/communication/roundtripper.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/communication/roundtripper.go b/internal/communication/roundtripper.go index 58de6f0..1dbaf5b 100644 --- a/internal/communication/roundtripper.go +++ b/internal/communication/roundtripper.go @@ -6,10 +6,13 @@ package communication import ( + "errors" "net/http" "strings" ) +const maxHeaders = 1000 + // RoundTripper is a simple type that wraps the default net/communication RoundTripper to add additional headers. type RoundTripper struct { Headers []string @@ -26,6 +29,10 @@ func NewRoundTripper(headers []string, transport *http.Transport) *RoundTripper // RoundTrip This simply adds our default headers to the request before passing it on to the default RoundTripper. func (roundTripper *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if len(req.Header) > maxHeaders { + return nil, errors.New("request includes too many headers") + } + newRequest := new(http.Request) *newRequest = *req newRequest.Header = make(http.Header, len(req.Header)) From ab78f210bf04a865fd5ce6bbc8bc6d168eff3e0c Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 21 Mar 2025 14:46:49 -0600 Subject: [PATCH 123/136] Upgraded go to 1.23.8 and golang.org/x/sync to v0.12.0 --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 5943bf6..3fd8dff 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,13 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.23.0 +go 1.23.8 require ( github.com/nginx/nginx-plus-go-client/v2 v2.3.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.11.0 + golang.org/x/sync v0.12.0 k8s.io/api v0.32.2 k8s.io/apimachinery v0.32.2 k8s.io/client-go v0.32.2 diff --git a/go.sum b/go.sum index e03c187..9169172 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From afb802e4cd8e541ba8db28125693b9524f05e343 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Fri, 21 Mar 2025 14:46:49 -0600 Subject: [PATCH 124/136] Bumped version to 1.1.2 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 524cb55..45a1b3f 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.1 +1.1.2 From 55f50e2f7094a3492d9520b751bbaaef754d9892 Mon Sep 17 00:00:00 2001 From: Aniruddh Kuthiala Date: Tue, 22 Apr 2025 12:51:31 -0600 Subject: [PATCH 125/136] NLB-6372: enable scanning on merge requests, merged to main and tags with v* --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7f73a13..0ebf127 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -154,7 +154,7 @@ whitesource-scan: rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' when: never - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - !reference [.whitesource-template-go, rules] artifacts: when: always paths: @@ -178,7 +178,6 @@ checkmarx-scan: CX_SOURCES: "." CX_FLOW_ZIP_EXCLUDE: "" needs: [] - allow_failure: true # https://docs.gitlab.com/ee/user/application_security/container_scanning/ container_scanning: From 8765185f5771a83f0370457a894886ff93ab0528 Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 19 May 2025 16:08:00 -0700 Subject: [PATCH 126/136] Enable dependency caching --- .gitlab-ci.yml | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ebf127..41c7290 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,23 +33,22 @@ workflow: .import-devops-core-services: &import-devops-core-services | source ${CI_PROJECT_DIR}/.devops.sh -# Disabld due to: NLB-5341 -# .go-cache: -# variables: -# GOPATH: $CI_PROJECT_DIR/.go -# cache: -# key: -# files: -# - go.mod -# - go.sum -# paths: -# - .go/pkg/mod/ +.go-cache: + variables: + GOPATH: $CI_PROJECT_DIR/.go + cache: + key: + files: + - go.mod + - go.sum + paths: + - .go/pkg/mod/ -# .go-cache-readonly: -# extends: -# - .go-cache -# cache: -# policy: pull +.go-cache-readonly: + extends: + - .go-cache + cache: + policy: pull .golang-private: &golang-private - | @@ -66,7 +65,7 @@ workflow: image: $DEVTOOLS_IMG extends: - .default-runner-large - # - .go-cache-readonly + - .go-cache-readonly script: - *golang-private - time make test @@ -85,7 +84,7 @@ lint + unit-test + build: image: $DEVTOOLS_IMG extends: - .devops-docker-cicd-large - # - .go-cache + - .go-cache script: - *golang-private - | @@ -143,7 +142,7 @@ whitesource-scan: stage: lint+test+build extends: - .default-runner - # - .go-cache + - .go-cache - .whitesource-template-go variables: PRODUCT_PREFIX: "n4a" From 1633c2151fd69d07ef5af16202e8f77dbc696b9d Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 19 May 2025 16:11:53 -0700 Subject: [PATCH 127/136] Skip go-cache while linting Go cache in the CI is seeded in the project working directory. We should skip the mod cache from lint/formatting as it's upstream code and there are high chances of the linting failing as upstream lint rules != our lint rules. --- .golangci.yml | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 6f0fddf..268e39f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -63,6 +63,8 @@ linters-settings: - "gitlab.com/f5/nginx/nginxazurelb/azure-resource-provider/internal/dpo/agent/certificates.CertGetRequest" issues: + exclude-dirs: + - .go/pkg/mod # Exclude configuration exclude-rules: # Exclude gochecknoinits and gosec from running on tests files diff --git a/Makefile b/Makefile index 354bd95..dd02b98 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ fmt: @find . -type f -name "*.go" -exec goimports -e -w {} \+ lint: - @find . -type f -name "*.go" -exec goimports -e -w {} \+ + @find . -type f -not -path "./.go/pkg/mod/*" -name "*.go" -exec goimports -e -w {} \+ @golangci-lint run -v ./... helm-lint: From c992734fe4482d8cba80b24b58644ffceb3d514c Mon Sep 17 00:00:00 2001 From: sarna Date: Mon, 19 May 2025 16:19:06 -0700 Subject: [PATCH 128/136] Exit on diff failures from goimports fixes --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index dd02b98..c45e2cf 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ fmt: lint: @find . -type f -not -path "./.go/pkg/mod/*" -name "*.go" -exec goimports -e -w {} \+ + @ git diff --exit-code @golangci-lint run -v ./... helm-lint: From 3ab6be15c408c6a1a0509941f6fdefd95be86752 Mon Sep 17 00:00:00 2001 From: Aniruddh Kuthiala Date: Fri, 30 May 2025 10:37:28 -0600 Subject: [PATCH 129/136] unit test flake fix and go version upgrade to 1.24.3 --- go.mod | 35 +++++++++-------- go.sum | 73 +++++++++++++++++++----------------- internal/probation/server.go | 12 +++++- version | 2 +- 4 files changed, 66 insertions(+), 56 deletions(-) diff --git a/go.mod b/go.mod index 3fd8dff..c09a4b4 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,16 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.23.8 +go 1.24.3 require ( github.com/nginx/nginx-plus-go-client/v2 v2.3.0 github.com/spf13/viper v1.19.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 golang.org/x/sync v0.12.0 - k8s.io/api v0.32.2 - k8s.io/apimachinery v0.32.2 - k8s.io/client-go v0.32.2 + k8s.io/api v0.33.1 + k8s.io/apimachinery v0.33.1 + k8s.io/client-go v0.33.1 ) require ( @@ -26,10 +26,8 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect + github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -54,22 +52,23 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.7.0 // indirect - google.golang.org/protobuf v1.35.1 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 9169172..80d0e9d 100644 --- a/go.sum +++ b/go.sum @@ -25,16 +25,12 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -80,8 +76,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -99,14 +95,16 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -115,6 +113,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -128,10 +128,10 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -140,16 +140,16 @@ golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -160,8 +160,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -174,21 +174,24 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.2 h1:bZrMLEkgizC24G9eViHGOPbW+aRo9duEISRIJKfdJuw= -k8s.io/api v0.32.2/go.mod h1:hKlhk4x1sJyYnHENsrdCWw31FEmCijNGPJO5WzHiJ6Y= -k8s.io/apimachinery v0.32.2 h1:yoQBR9ZGkA6Rgmhbp/yuT9/g+4lxtsGYwW6dR6BDPLQ= -k8s.io/apimachinery v0.32.2/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/client-go v0.32.2 h1:4dYCD4Nz+9RApM2b/3BtVvBHw54QjMFUl1OLcJG5yOA= -k8s.io/client-go v0.32.2/go.mod h1:fpZ4oJXclZ3r2nDOv+Ux3XcJutfrwjKTCHz2H3sww94= +k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= +k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= +k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= +k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= +k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= +sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/internal/probation/server.go b/internal/probation/server.go index 9520fc7..c3328c7 100644 --- a/internal/probation/server.go +++ b/internal/probation/server.go @@ -8,6 +8,7 @@ package probation import ( "fmt" "log/slog" + "net" "net/http" "time" ) @@ -58,10 +59,17 @@ func (hs *HealthServer) Start() { mux.HandleFunc("/livez", hs.HandleLive) mux.HandleFunc("/readyz", hs.HandleReady) mux.HandleFunc("/startupz", hs.HandleStartup) - hs.httpServer = &http.Server{Addr: address, Handler: mux, ReadTimeout: 2 * time.Second} + + listener, err := net.Listen("tcp", address) + if err != nil { + slog.Error("failed to listen", "error", err) + return + } + + hs.httpServer = &http.Server{Handler: mux, ReadTimeout: 2 * time.Second} go func() { - if err := hs.httpServer.ListenAndServe(); err != nil { + if err := hs.httpServer.Serve(listener); err != nil { slog.Error("unable to start probe listener", "address", hs.httpServer.Addr, "error", err) } }() diff --git a/version b/version index 45a1b3f..781dcb0 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.2 +1.1.3 From f67d455811373caa6e2bf1d248fb35900d053706 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 26 Jun 2025 15:32:04 -0600 Subject: [PATCH 130/136] Fixed up helm templating for nginx-hosts param to support multiple nginx hosts In order for the nginx-hosts yaml field to be parsed correctly by viper the template needs to: 1. not put double quotes around the value (this causes viper to interpret it as a string) 2. render it as a JSON array rather than a go representation of a slice. --- charts/nlk/templates/nlk-configmap.yaml | 2 +- internal/configuration/configuration_test.go | 112 +++++++++++++----- .../testdata/multiple-nginx-hosts.yaml | 11 ++ .../testdata/one-nginx-host.yaml | 11 ++ internal/configuration/testdata/test.yaml | 11 -- version | 2 +- 6 files changed, 105 insertions(+), 44 deletions(-) create mode 100644 internal/configuration/testdata/multiple-nginx-hosts.yaml create mode 100644 internal/configuration/testdata/one-nginx-host.yaml delete mode 100644 internal/configuration/testdata/test.yaml diff --git a/charts/nlk/templates/nlk-configmap.yaml b/charts/nlk/templates/nlk-configmap.yaml index 38475a9..0b6db57 100644 --- a/charts/nlk/templates/nlk-configmap.yaml +++ b/charts/nlk/templates/nlk-configmap.yaml @@ -9,7 +9,7 @@ data: log-level: "{{ . }}" {{- end }} {{- with .Values.nlk.config.nginxHosts }} - nginx-hosts: "{{ . }}" + nginx-hosts: {{ toJson . }} {{- end }} tls-mode: "{{ .Values.nlk.config.tls.mode }}" {{- with .Values.nlk.config.serviceAnnotationMatch }} diff --git a/internal/configuration/configuration_test.go b/internal/configuration/configuration_test.go index 9694989..fc41974 100644 --- a/internal/configuration/configuration_test.go +++ b/internal/configuration/configuration_test.go @@ -12,41 +12,91 @@ import ( func TestConfiguration(t *testing.T) { t.Parallel() - expectedSettings := configuration.Settings{ - LogLevel: "warn", - NginxPlusHosts: []string{"https://10.0.0.1:9000/api"}, - TLSMode: configuration.NoTLS, - Certificates: &certification.Certificates{ - CaCertificateSecretKey: "fakeCAKey", - ClientCertificateSecretKey: "fakeCertKey", - }, - Handler: configuration.HandlerSettings{ - RetryCount: 5, - Threads: 1, - WorkQueueSettings: configuration.WorkQueueSettings{ - RateLimiterBase: time.Second * 2, - RateLimiterMax: time.Second * 60, - Name: "nlk-handler", + + tests := map[string]struct { + testFile string + expectedSettings configuration.Settings + }{ + "one nginx plus host": { + testFile: "one-nginx-host", + expectedSettings: configuration.Settings{ + LogLevel: "warn", + NginxPlusHosts: []string{"https://10.0.0.1:9000/api"}, + TLSMode: configuration.NoTLS, + Certificates: &certification.Certificates{ + CaCertificateSecretKey: "fakeCAKey", + ClientCertificateSecretKey: "fakeCertKey", + }, + Handler: configuration.HandlerSettings{ + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-handler", + }, + }, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + Watcher: configuration.WatcherSettings{ + ResyncPeriod: 0, + ServiceAnnotation: "fakeServiceMatch", + }, }, }, - Synchronizer: configuration.SynchronizerSettings{ - MaxMillisecondsJitter: 750, - MinMillisecondsJitter: 250, - RetryCount: 5, - Threads: 1, - WorkQueueSettings: configuration.WorkQueueSettings{ - RateLimiterBase: time.Second * 2, - RateLimiterMax: time.Second * 60, - Name: "nlk-synchronizer", + "multiple nginx plus hosts": { + testFile: "multiple-nginx-hosts", + expectedSettings: configuration.Settings{ + LogLevel: "warn", + NginxPlusHosts: []string{"https://10.0.0.1:9000/api", "https://10.0.0.2:9000/api"}, + TLSMode: configuration.NoTLS, + Certificates: &certification.Certificates{ + CaCertificateSecretKey: "fakeCAKey", + ClientCertificateSecretKey: "fakeCertKey", + }, + Handler: configuration.HandlerSettings{ + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-handler", + }, + }, + Synchronizer: configuration.SynchronizerSettings{ + MaxMillisecondsJitter: 750, + MinMillisecondsJitter: 250, + RetryCount: 5, + Threads: 1, + WorkQueueSettings: configuration.WorkQueueSettings{ + RateLimiterBase: time.Second * 2, + RateLimiterMax: time.Second * 60, + Name: "nlk-synchronizer", + }, + }, + Watcher: configuration.WatcherSettings{ + ResyncPeriod: 0, + ServiceAnnotation: "fakeServiceMatch", + }, }, }, - Watcher: configuration.WatcherSettings{ - ResyncPeriod: 0, - ServiceAnnotation: "fakeServiceMatch", - }, } - settings, err := configuration.Read("test", "./testdata") - require.NoError(t, err) - require.Equal(t, expectedSettings, settings) + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + settings, err := configuration.Read(tc.testFile, "./testdata") + require.NoError(t, err) + require.Equal(t, tc.expectedSettings, settings) + }) + } } diff --git a/internal/configuration/testdata/multiple-nginx-hosts.yaml b/internal/configuration/testdata/multiple-nginx-hosts.yaml new file mode 100644 index 0000000..2235c1a --- /dev/null +++ b/internal/configuration/testdata/multiple-nginx-hosts.yaml @@ -0,0 +1,11 @@ +ca-certificate: "fakeCAKey" +client-certificate: "fakeCertKey" +log-level: "warn" +nginx-hosts: ["https://10.0.0.1:9000/api", "https://10.0.0.2:9000/api"] +tls-mode: "no-tls" +service-annotation-match: "fakeServiceMatch" +creationTimestamp: "2024-09-04T17:59:20Z" +name: "nlk-config" +namespace: "nlk" +resourceVersion: "5909" +uid: "66d49974-49d6-4ad8-8135-dcebda7b5c9e" diff --git a/internal/configuration/testdata/one-nginx-host.yaml b/internal/configuration/testdata/one-nginx-host.yaml new file mode 100644 index 0000000..f05d81e --- /dev/null +++ b/internal/configuration/testdata/one-nginx-host.yaml @@ -0,0 +1,11 @@ +ca-certificate: "fakeCAKey" +client-certificate: "fakeCertKey" +log-level: "warn" +nginx-hosts: "https://10.0.0.1:9000/api" +tls-mode: "no-tls" +service-annotation-match: "fakeServiceMatch" +creationTimestamp: "2024-09-04T17:59:20Z" +name: "nlk-config" +namespace: "nlk" +resourceVersion: "5909" +uid: "66d49974-49d6-4ad8-8135-dcebda7b5c9e" diff --git a/internal/configuration/testdata/test.yaml b/internal/configuration/testdata/test.yaml deleted file mode 100644 index 717dcdb..0000000 --- a/internal/configuration/testdata/test.yaml +++ /dev/null @@ -1,11 +0,0 @@ -ca-certificate: fakeCAKey -client-certificate: fakeCertKey -log-level: warn -nginx-hosts: https://10.0.0.1:9000/api -tls-mode: no-tls -service-annotation-match: fakeServiceMatch -creationTimestamp: "2024-09-04T17:59:20Z" -name: nlk-config -namespace: nlk -resourceVersion: "5909" -uid: 66d49974-49d6-4ad8-8135-dcebda7b5c9e diff --git a/version b/version index 781dcb0..65087b4 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.3 +1.1.4 From c4163b2e14dbad4c5da6534e8875c3296562cd28 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Mon, 30 Jun 2025 14:20:12 -0600 Subject: [PATCH 131/136] NLB-6295 Simplified user configuration of TLS modes The biggest change here is to remove most the TLS modes to enable mTLS and self-signed certificates. Product decided that this was too complex and there was not enough user demand for most of these options. We decided to pare down the code and remove tests that were no longer well maintained. The remaining configuration allows users to toggle a single switch: whether to make the http client verify the NGINX host's certificate chain and host name if https is being used. If the user wishes to enable https with self-signed certs they can use the "skip-verify-tls" setting to allow this. The default behavior is to perform this verification. We are maintaining the deprecated "no-tls" and "ca-tls" inputs for NGINXaaS backwards comptability reasons. The "no-tls" setting name was highly misleading, because all it did was disable TLS verification: it DID NOT disable TLS altogether in https mode. Similarly, the "ca-tls" setting did not enable TLS itself. TLS is enabled by default when the URL of the NGINX host includes the https protocol. The user setting merely enforced the verification of the certificate chain and host as described above. --- cmd/certificates-test-harness/doc.go | 11 - cmd/certificates-test-harness/main.go | 81 ---- cmd/tls-config-factory-test-harness/doc.go | 1 - cmd/tls-config-factory-test-harness/main.go | 236 ---------- internal/authentication/doc.go | 10 - internal/authentication/factory.go | 123 ----- internal/authentication/factory_test.go | 442 ------------------ internal/certification/certificates.go | 215 --------- internal/certification/certificates_test.go | 232 --------- internal/certification/doc.go | 10 - internal/communication/factory.go | 33 +- internal/communication/factory_test.go | 15 +- internal/communication/roundtripper_test.go | 12 +- internal/configuration/configuration_test.go | 60 ++- internal/configuration/settings.go | 39 +- .../testdata/one-nginx-host.yaml | 1 - internal/configuration/tlsmodes.go | 48 +- internal/configuration/tlsmodes_test.go | 76 --- internal/core/secret_bytes.go | 21 - internal/core/secret_bytes_test.go | 33 -- internal/synchronization/synchronizer.go | 3 +- 21 files changed, 95 insertions(+), 1607 deletions(-) delete mode 100644 cmd/certificates-test-harness/doc.go delete mode 100644 cmd/certificates-test-harness/main.go delete mode 100644 cmd/tls-config-factory-test-harness/doc.go delete mode 100644 cmd/tls-config-factory-test-harness/main.go delete mode 100644 internal/authentication/doc.go delete mode 100644 internal/authentication/factory.go delete mode 100644 internal/authentication/factory_test.go delete mode 100644 internal/certification/certificates.go delete mode 100644 internal/certification/certificates_test.go delete mode 100644 internal/certification/doc.go delete mode 100644 internal/configuration/tlsmodes_test.go delete mode 100644 internal/core/secret_bytes.go delete mode 100644 internal/core/secret_bytes_test.go diff --git a/cmd/certificates-test-harness/doc.go b/cmd/certificates-test-harness/doc.go deleted file mode 100644 index 2d76fd5..0000000 --- a/cmd/certificates-test-harness/doc.go +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -/* -Package certificates_test_harness includes functionality boostrap -and test the certification.Certificates implplementation. -*/ - -package main diff --git a/cmd/certificates-test-harness/main.go b/cmd/certificates-test-harness/main.go deleted file mode 100644 index 898c3a3..0000000 --- a/cmd/certificates-test-harness/main.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "log/slog" - "os" - "path/filepath" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/util/homedir" -) - -func main() { - handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) - logger := slog.New(handler) - slog.SetDefault(logger) - err := run() - if err != nil { - slog.Error(err.Error()) - os.Exit(1) - } -} - -func run() error { - slog.Info("certificates-test-harness::run") - - ctx := context.Background() - var err error - - k8sClient, err := buildKubernetesClient() - if err != nil { - return fmt.Errorf(`error building a Kubernetes client: %w`, err) - } - - certificates := certification.NewCertificates(k8sClient, nil) - - err = certificates.Initialize() - if err != nil { - return fmt.Errorf(`error occurred initializing certificates: %w`, err) - } - - go certificates.Run(ctx) //nolint:errcheck - - <-ctx.Done() - return nil -} - -func buildKubernetesClient() (*kubernetes.Clientset, error) { - slog.Debug("Watcher::buildKubernetesClient") - - var kubeconfig *string - var k8sConfig *rest.Config - - k8sConfig, err := rest.InClusterConfig() - if errors.Is(err, rest.ErrNotInCluster) { - if home := homedir.HomeDir(); home != "" { - path := filepath.Join(home, ".kube", "config") - kubeconfig = &path - - k8sConfig, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) - if err != nil { - return nil, fmt.Errorf(`error occurred building the kubeconfig: %w`, err) - } - } else { - return nil, fmt.Errorf(`not running in a Cluster: %w`, err) - } - } else if err != nil { - return nil, fmt.Errorf(`error occurred getting the Cluster config: %w`, err) - } - - client, err := kubernetes.NewForConfig(k8sConfig) - if err != nil { - return nil, fmt.Errorf(`error occurred creating a client: %w`, err) - } - return client, nil -} diff --git a/cmd/tls-config-factory-test-harness/doc.go b/cmd/tls-config-factory-test-harness/doc.go deleted file mode 100644 index 06ab7d0..0000000 --- a/cmd/tls-config-factory-test-harness/doc.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/cmd/tls-config-factory-test-harness/main.go b/cmd/tls-config-factory-test-harness/main.go deleted file mode 100644 index 638b0bc..0000000 --- a/cmd/tls-config-factory-test-harness/main.go +++ /dev/null @@ -1,236 +0,0 @@ -package main - -import ( - "bufio" - "log/slog" - "os" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -const ( - CaCertificateSecretKey = "nlk-tls-ca-secret" - ClientCertificateSecretKey = "nlk-tls-client-secret" -) - -type TLSConfiguration struct { - Description string - Settings configuration.Settings -} - -func main() { - handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}) - logger := slog.New(handler) - slog.SetDefault(logger) - - configurations := buildConfigMap() - - for name, settings := range configurations { - slog.Info("\033[H\033[2J") - - slog.Info("\n\n\t*** Building TLS config\n\n", "name", name) - - tlsConfig, err := authentication.NewTLSConfig(settings.Settings) - if err != nil { - panic(err) - } - - rootCaCount := 0 - certificateCount := 0 - - if tlsConfig.RootCAs != nil { - rootCaCount = len(tlsConfig.RootCAs.Subjects()) //nolint:staticcheck - } - - if tlsConfig.Certificates != nil { - certificateCount = len(tlsConfig.Certificates) - } - - slog.Info("Successfully built TLS config", - "description", settings.Description, - "rootCaCount", rootCaCount, - "certificateCount", certificateCount, - ) - - _, _ = bufio.NewReader(os.Stdin).ReadBytes('\n') - } - - slog.Info("\033[H\033[2J") - slog.Info("\n\n\t*** All done! ***\n\n") -} - -func buildConfigMap() map[string]TLSConfiguration { - configurations := make(map[string]TLSConfiguration) - - configurations["ss-tls"] = TLSConfiguration{ - Description: "Self-signed TLS requires just a CA certificate", - Settings: ssTLSConfig(), - } - - configurations["ss-mtls"] = TLSConfiguration{ - Description: "Self-signed mTLS requires a CA certificate and a client certificate", - Settings: ssMtlsConfig(), - } - - configurations["ca-tls"] = TLSConfiguration{ - Description: "CA TLS requires no certificates", - Settings: caTLSConfig(), - } - - configurations["ca-mtls"] = TLSConfiguration{ - Description: "CA mTLS requires a client certificate", - Settings: caMtlsConfig(), - } - - return configurations -} - -func ssTLSConfig() configuration.Settings { - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - - return configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: certificates, - } -} - -func ssMtlsConfig() configuration.Settings { - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - - return configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: certificates, - } -} - -func caTLSConfig() configuration.Settings { - return configuration.Settings{ - TLSMode: configuration.CertificateAuthorityTLS, - } -} - -func caMtlsConfig() configuration.Settings { - certs := make(map[string]map[string]core.SecretBytes) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - - return configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: certificates, - } -} - -func caCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIDTzCCAjcCFA4Zdj3E9TdjOP48eBRDGRLfkj7CMA0GCSqGSIb3DQEBCwUAMGQx -CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0 -dGxlMQ4wDAYDVQQKDAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFu -Y2VzMB4XDTIzMDkyOTE3MTY1MVoXDTIzMTAyOTE3MTY1MVowZDELMAkGA1UEBhMC -VVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxDjAMBgNV -BAoMBU5HSU5YMR4wHAYDVQQLDBVDb21tdW5pdHkgJiBBbGxpYW5jZXMwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwlI4ZvJ/6hvqULFVL+1ZSRDTPQ48P -umehJhPz6xPhC9UkeTe2FZxm2Rsi1I5QXm/bTG2OcX775jgXzae9NQjctxwrz4Ks -LOWUvRkkfhQR67xk0Noux76/9GWGnB+Fapn54tlWql6uHQfOu1y7MCRkZ27zHbkk -lq4Oa2RmX8rIyECWgbTyL0kETBVJU8bYORQ5JjhRlz08inq3PggY8blrehIetrWN -dw+gzcqdvAI2uSCodHTHM/77KipnYmPiSiDjSDRlXdxTG8JnyIB78IoH/sw6RyBm -CvVa3ytvKziXAvbBoXq5On5WmMRF97p/MmBc53ExMuDZjA4fisnViS0PAgMBAAEw -DQYJKoZIhvcNAQELBQADggEBAJeoa2P59zopLjBInx/DnWn1N1CmFLb0ejKxG2jh -cOw15Sx40O0XrtrAto38iu4R/bkBeNCSUILlT+A3uYDila92Dayvls58WyIT3meD -G6+Sx/QDF69+4AXpVy9mQ+hxcofpFA32+GOMXwmk2OrAcdSkkGSBhZXgvTpQ64dl -xSiQ5EQW/K8LoBoEOXfjIZJNPORgKn5MI09AY7/47ycKDKTUU2yO8AtIHYKttw0x -kfIg7QOdo1F9IXVpGjJI7ynyrgsCEYxMoDyH42Dq84eKgrUFLEXemEz8hgdFgK41 -0eUYhAtzWHbRPBp+U/34CQoZ5ChNFp2YipvtXrzKE8KLkuM= ------END CERTIFICATE----- -` -} - -func clientCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIEDDCCAvSgAwIBAgIULDFXwGrTohN/PRao2rSLk9VxFdgwDQYJKoZIhvcNAQEL -BQAwXTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEjAQBgNVBAcM -CUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVyMRQwEgYDVQQLDAtEZXZlbG9wbWVu -dDAeFw0yMzA5MjkxNzA3NTRaFw0yNDA5MjgxNzA3NTRaMGQxCzAJBgNVBAYTAlVT -MRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMQ4wDAYDVQQK -DAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFuY2VzMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqNuEZ6+TcFrmzcwp8u8mzk0jPd47GKk -H9wwdkFCzGdd8KJkFQhzLyimZIWkRDYmhaxZd76jKGBpdfyivR4e4Mi5WYlpPGMI -ppM7/rMYP8yn04tkokAazbqjOTlF8NUKqGQwqAN4Z/PvoG2HyP9omGpuLWTbjKto -oGr5aPBIhzlICU3OjHn6eKaekJeAYBo3uQFYOxCjtE9hJLDOY4q7zomMJfYoeoA2 -Afwkx1Lmozp2j/esB52/HlCKVhAOzZsPzM+E9eb1Q722dUed4OuiVYSfrDzeImrA -TufzTBTMEpFHCtdBGocZ3LRd9qmcP36ZCMsJNbYnQZV3XsI4JhjjHwIDAQABo4G8 -MIG5MBMGA1UdJQQMMAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRDl4jeiE1mJDPrYmQx -g2ndkWxpYjCBggYDVR0jBHsweaFhpF8wXTELMAkGA1UEBhMCVVMxEzARBgNVBAgM -Cldhc2hpbmd0b24xEjAQBgNVBAcMCUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVy -MRQwEgYDVQQLDAtEZXZlbG9wbWVudIIUNxx2Mr+PKXiF3d2i51fb/rnWbBgwDQYJ -KoZIhvcNAQELBQADggEBAL0wS6LkFuqGDlhaTGnAXRwRDlC6uwrm8wNWppaw9Vqt -eaZGFzodcCFp9v8jjm1LsTv7gEUBnWtn27LGP4GJSpZjiq6ulJypBxo/G0OkMByK -ky4LeGY7/BQzjzHdfXEq4gwfC45ni4n54uS9uzW3x+AwLSkxPtBxSwxhtwBLo9aE -Ql4rHUoWc81mhGO5mMZBaorxZXps1f3skfP+wZX943FIMt5gz4hkxwFp3bI/FrqH -R8DLUlCzBA9+7WIFD1wi25TV+Oyq3AjT/KiVmR+umrukhnofCWe8JiVpb5iJcd2k -Rc7+bvyb5OCnJdEX08XGWmF2/OFKLrCzLH1tQxk7VNE= ------END CERTIFICATE----- -` -} - -// clientKeyPEM returns a PEM-encoded client key. -// Note: The key is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func clientKeyPEM() string { - return ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCio24Rnr5NwWub -NzCny7ybOTSM93jsYqQf3DB2QULMZ13womQVCHMvKKZkhaRENiaFrFl3vqMoYGl1 -/KK9Hh7gyLlZiWk8Ywimkzv+sxg/zKfTi2SiQBrNuqM5OUXw1QqoZDCoA3hn8++g -bYfI/2iYam4tZNuMq2igavlo8EiHOUgJTc6Mefp4pp6Ql4BgGje5AVg7EKO0T2Ek -sM5jirvOiYwl9ih6gDYB/CTHUuajOnaP96wHnb8eUIpWEA7Nmw/Mz4T15vVDvbZ1 -R53g66JVhJ+sPN4iasBO5/NMFMwSkUcK10EahxnctF32qZw/fpkIywk1tidBlXde -wjgmGOMfAgMBAAECggEAA+R2b2yFsHW3HhVhkDqDjpF9bPxFRB8OP4b1D/d64kp9 -CJPSYmB75T6LUO+T4WAMZvmbgI6q9/3quDyuJmmQop+bNAXiY2QZYmc2sd9Wbrx2 -rczxwSJYoeDcJDP3NQ7cPPB866B9ortHWmcUr15RgghWD7cQvBqkG+bDhlvt2HKg -NZmL6R0U1bVAlRMtFJiEdMHuGnPmoDU5IGc1fKjsgijLeMboUrEaXWINoEm8ii5e -/mnsfLCBmeJAsKuXxL8/1UmvWYE/ltDfYBVclKhcH2UWTZv7pdRtHnu49lkZivUB -ZvH2DHsSMjXj6+HHr6RcRGmnMDyfhJFPCjOdTjf4oQKBgQDeYLWZx22zGXgfb7md -MhdKed9GxMJHzs4jDouqrHy0w95vwMi7RXgeKpKXiCruqSEB/Trtq01f7ekh0mvJ -Ys0h4A5tkrT5BVVBs+65uF/kSF2z/CYGNRhAABO7UM+B1e3tlnjfjeb/M78IcFbT -FyBN90A/+a9JGZ4obt3ack3afwKBgQC7OncnXC9L5QCWForJWQCNO3q3OW1Gaoxe -OAnmnPSJ7NUd7xzDNE8pzBUWXysZCoRU3QNElcQfzHWtZx1iqJPk3ERK2awNsnV7 -X2Fu4vHzIr5ZqVnM8NG7+iWrxRLf+ctcEvPiqRYo+g+r5tTGJqWh2nh9W7iQwwwE -1ikoxFBnYQKBgCbDdOR5fwXZSrcwIorkUGsLE4Cii7s4sXYq8u2tY4+fFQcl89ex -JF8dzK/dbJ5tnPNb0Qnc8n/mWN0scN2J+3gMNnejOyitZU8urk5xdUW115+oNHig -iLmfSdE9JO7c+7yOnkNZ2QpjWsl9y6TAQ0FT+D8upv93F7q0mLebdTbBAoGBALmp -r5EThD9RlvQ+5F/oZ3imO/nH88n5TLr9/St4B7NibLAjdrVIgRwkqeCmfRl26WUy -SdRQY81YtnU/JM+59fbkSsCi/FAU4RV3ryoD2QRPNs249zkYshMjawncAuyiS/xB -OyJQpI3782B3JhZdKrDG8eb19p9vG9MMAILRsh3hAoGASCvmq10nHHGFYTerIllQ -sohNaw3KDlQTkpyOAztS4jOXwvppMXbYuCznuJbHz0NEM2ww+SiA1RTvD/gosYYC -mMgqRga/Qu3b149M3wigDjK+RAcyuNGZN98bqU/UjJLjqH6IMutt59+9XNspcD96 -z/3KkMx4uqJXZyvQrmkolSg= ------END PRIVATE KEY----- -` -} - -func buildClientCertificateEntry(keyPEM, certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - certification.CertificateKeyKey: core.SecretBytes([]byte(keyPEM)), - } -} - -func buildCaCertificateEntry(certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - } -} diff --git a/internal/authentication/doc.go b/internal/authentication/doc.go deleted file mode 100644 index 109255e..0000000 --- a/internal/authentication/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -/* -Package authentication includes functionality to secure communications between NLK and NGINX Plus hosts. -*/ - -package authentication diff --git a/internal/authentication/factory.go b/internal/authentication/factory.go deleted file mode 100644 index c0664f6..0000000 --- a/internal/authentication/factory.go +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - * - * Factory for creating tls.Config objects based on the provided `tls-mode`. - */ - -package authentication - -import ( - "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" - "log/slog" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" -) - -func NewTLSConfig(settings configuration.Settings) (*tls.Config, error) { - slog.Debug("authentication::NewTLSConfig Creating TLS config", "mode", settings.TLSMode) - switch settings.TLSMode { - - case configuration.NoTLS: - return buildBasicTLSConfig(true), nil - - case configuration.SelfSignedTLS: // needs ca cert - return buildSelfSignedTLSConfig(settings.Certificates) - - case configuration.SelfSignedMutualTLS: // needs ca cert and client cert - return buildSelfSignedMtlsConfig(settings.Certificates) - - case configuration.CertificateAuthorityTLS: // needs nothing - return buildBasicTLSConfig(false), nil - - case configuration.CertificateAuthorityMutualTLS: // needs client cert - return buildCATLSConfig(settings.Certificates) - - default: - return nil, fmt.Errorf("unknown TLS mode: %s", settings.TLSMode) - } -} - -func buildSelfSignedTLSConfig(certificates *certification.Certificates) (*tls.Config, error) { - slog.Debug("authentication::buildSelfSignedTlsConfig Building self-signed TLS config") - certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) - if err != nil { - return nil, err - } - - //nolint:gosec - return &tls.Config{ - InsecureSkipVerify: false, - RootCAs: certPool, - }, nil -} - -func buildSelfSignedMtlsConfig(certificates *certification.Certificates) (*tls.Config, error) { - slog.Debug("authentication::buildSelfSignedMtlsConfig Building self-signed mTLS config") - certPool, err := buildCaCertificatePool(certificates.GetCACertificate()) - if err != nil { - return nil, err - } - - certificate, err := buildCertificates(certificates.GetClientCertificate()) - if err != nil { - return nil, err - } - slog.Debug("buildSelfSignedMtlsConfig Certificate", "certificate", certificate) - - //nolint:gosec - return &tls.Config{ - InsecureSkipVerify: false, - RootCAs: certPool, - ClientAuth: tls.RequireAndVerifyClientCert, - Certificates: []tls.Certificate{certificate}, - }, nil -} - -func buildBasicTLSConfig(skipVerify bool) *tls.Config { - slog.Debug("authentication::buildBasicTLSConfig", slog.Bool("skipVerify", skipVerify)) - return &tls.Config{ - InsecureSkipVerify: skipVerify, //nolint:gosec - } -} - -func buildCATLSConfig(certificates *certification.Certificates) (*tls.Config, error) { - slog.Debug("authentication::buildCATLSConfig") - certificate, err := buildCertificates(certificates.GetClientCertificate()) - if err != nil { - return nil, err - } - - //nolint:gosec - return &tls.Config{ - InsecureSkipVerify: false, - Certificates: []tls.Certificate{certificate}, - }, nil -} - -func buildCertificates(privateKeyPEM []byte, certificatePEM []byte) (tls.Certificate, error) { - slog.Debug("authentication::buildCertificates") - return tls.X509KeyPair(certificatePEM, privateKeyPEM) -} - -func buildCaCertificatePool(caCert []byte) (*x509.CertPool, error) { - slog.Debug("authentication::buildCaCertificatePool") - block, _ := pem.Decode(caCert) - if block == nil { - return nil, fmt.Errorf("failed to decode PEM block containing CA certificate") - } - - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("error parsing certificate: %w", err) - } - - caCertPool := x509.NewCertPool() - caCertPool.AddCert(cert) - - return caCertPool, nil -} diff --git a/internal/authentication/factory_test.go b/internal/authentication/factory_test.go deleted file mode 100644 index 4777607..0000000 --- a/internal/authentication/factory_test.go +++ /dev/null @@ -1,442 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package authentication - -import ( - "testing" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -const ( - CaCertificateSecretKey = "nlk-tls-ca-secret" - ClientCertificateSecretKey = "nlk-tls-client-secret" -) - -func TestTlsFactory_UnspecifiedModeDefaultsToNoTls(t *testing.T) { - t.Parallel() - - tlsConfig, err := NewTLSConfig(configuration.Settings{}) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != true { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be true`) - } -} - -func TestTlsFactory_SelfSignedTlsMode(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: certificates, - } - - tlsConfig, err := NewTLSConfig(settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) != 0 { - t.Fatalf(`tlsConfig.Certificates should be empty`) - } - - if tlsConfig.RootCAs == nil { - t.Fatalf(`tlsConfig.RootCAs should not be nil`) - } -} - -func TestTlsFactory_SelfSignedTlsModeCertPoolError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "failed to decode PEM block containing CA certificate" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_SelfSignedTlsModeCertPoolCertificateParseError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificateDataPEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "error parsing certificate: x509: inner and outer signature algorithm identifiers don't match" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_SelfSignedMtlsMode(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: certificates, - } - - tlsConfig, err := NewTLSConfig(settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) == 0 { - t.Fatalf(`tlsConfig.Certificates should not be empty`) - } - - if tlsConfig.RootCAs == nil { - t.Fatalf(`tlsConfig.RootCAs should not be nil`) - } -} - -func TestTlsFactory_SelfSignedMtlsModeCertPoolError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(invalidCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "failed to decode PEM block containing CA certificate" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_SelfSignedMtlsModeClientCertificateError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.SelfSignedMutualTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "tls: failed to find any PEM data in certificate input" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -func TestTlsFactory_CaTlsMode(t *testing.T) { - t.Parallel() - settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityTLS, - } - - tlsConfig, err := NewTLSConfig(settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) != 0 { - t.Fatalf(`tlsConfig.Certificates should be empty`) - } - - if tlsConfig.RootCAs != nil { - t.Fatalf(`tlsConfig.RootCAs should be nil`) - } -} - -func TestTlsFactory_CaMtlsMode(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), clientCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: certificates, - } - - tlsConfig, err := NewTLSConfig(settings) - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } - - if tlsConfig == nil { - t.Fatalf(`tlsConfig should not be nil`) - } - - if tlsConfig.InsecureSkipVerify != false { - t.Fatalf(`tlsConfig.InsecureSkipVerify should be false`) - } - - if len(tlsConfig.Certificates) == 0 { - t.Fatalf(`tlsConfig.Certificates should not be empty`) - } - - if tlsConfig.RootCAs != nil { - t.Fatalf(`tlsConfig.RootCAs should be nil`) - } -} - -func TestTlsFactory_CaMtlsModeClientCertificateError(t *testing.T) { - t.Parallel() - certs := make(map[string]map[string]core.SecretBytes) - certs[CaCertificateSecretKey] = buildCaCertificateEntry(caCertificatePEM()) - certs[ClientCertificateSecretKey] = buildClientCertificateEntry(clientKeyPEM(), invalidCertificatePEM()) - - certificates := certification.NewCertificates(nil, certs) - certificates.CaCertificateSecretKey = CaCertificateSecretKey - certificates.ClientCertificateSecretKey = ClientCertificateSecretKey - - settings := configuration.Settings{ - TLSMode: configuration.CertificateAuthorityMutualTLS, - Certificates: certificates, - } - - _, err := NewTLSConfig(settings) - if err == nil { - t.Fatalf(`Expected an error`) - } - - if err.Error() != "tls: failed to find any PEM data in certificate input" { - t.Fatalf(`Unexpected error message: %v`, err) - } -} - -// caCertificatePEM returns a PEM-encoded CA certificate. -// Note: The certificate is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func caCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIDTzCCAjcCFA4Zdj3E9TdjOP48eBRDGRLfkj7CMA0GCSqGSIb3DQEBCwUAMGQx -CzAJBgNVBAYTAlVTMRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0 -dGxlMQ4wDAYDVQQKDAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFu -Y2VzMB4XDTIzMDkyOTE3MTY1MVoXDTIzMTAyOTE3MTY1MVowZDELMAkGA1UEBhMC -VVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxDjAMBgNV -BAoMBU5HSU5YMR4wHAYDVQQLDBVDb21tdW5pdHkgJiBBbGxpYW5jZXMwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwlI4ZvJ/6hvqULFVL+1ZSRDTPQ48P -umehJhPz6xPhC9UkeTe2FZxm2Rsi1I5QXm/bTG2OcX775jgXzae9NQjctxwrz4Ks -LOWUvRkkfhQR67xk0Noux76/9GWGnB+Fapn54tlWql6uHQfOu1y7MCRkZ27zHbkk -lq4Oa2RmX8rIyECWgbTyL0kETBVJU8bYORQ5JjhRlz08inq3PggY8blrehIetrWN -dw+gzcqdvAI2uSCodHTHM/77KipnYmPiSiDjSDRlXdxTG8JnyIB78IoH/sw6RyBm -CvVa3ytvKziXAvbBoXq5On5WmMRF97p/MmBc53ExMuDZjA4fisnViS0PAgMBAAEw -DQYJKoZIhvcNAQELBQADggEBAJeoa2P59zopLjBInx/DnWn1N1CmFLb0ejKxG2jh -cOw15Sx40O0XrtrAto38iu4R/bkBeNCSUILlT+A3uYDila92Dayvls58WyIT3meD -G6+Sx/QDF69+4AXpVy9mQ+hxcofpFA32+GOMXwmk2OrAcdSkkGSBhZXgvTpQ64dl -xSiQ5EQW/K8LoBoEOXfjIZJNPORgKn5MI09AY7/47ycKDKTUU2yO8AtIHYKttw0x -kfIg7QOdo1F9IXVpGjJI7ynyrgsCEYxMoDyH42Dq84eKgrUFLEXemEz8hgdFgK41 -0eUYhAtzWHbRPBp+U/34CQoZ5ChNFp2YipvtXrzKE8KLkuM= ------END CERTIFICATE----- -` -} - -func invalidCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIClzCCAX+gAwIBAgIJAIfPhC0RG6CwMA0GCSqGSIb3DQEBCwUAMBkxFzAVBgNV -BAMMDm9pbCBhdXRob3JpdHkwHhcNMjAwNDA3MTUwOTU1WhcNMjEwNDA2MTUwOTU1 -WjBMMSAwHgYDVQQLDBd5b3VuZy1jaGFsbGVuZ2UgdGVzdCBjb25zdW1lczEfMB0G -A1UECwwWc28wMS5jb3Jwb3JhdGlvbnNvY2lhbDEhMB8GA1UEAwwYc29tMS5jb3Jw -b3JhdGlvbnNvY2lhbC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB -AQDGRX31uzy+yLUOz7wOJHHm2dzrDgUbC6RZDjURvZxyt2Zi5wYWsEB5r5YhN7L0 -y1R9f+MGwNITIz9nYZuU/PLFOvzF5qX7A8TbdgjZEqvXe2NZ9J2z3iWvYQLN8Py3 -nv/Y6wadgXEBRCNNuIg/bQ9XuOr9tfB6j4Ut1GLU0eIlV/L3Rf9Y6SgrAl+58ITj -Wrg3Js/Wz3J2JU4qBD8U4I3XvUyfnX2SAG8Llm4KBuYz7g63Iu05s6RnmG+Xhu2T -5f2DWZUeATWbAlUW/M4NLO1+5H0gOr0TGulETQ6uElMchT7s/H6Rv1CV+CNCCgEI -adRjWJq9yQ+KrE+urSMCXu8XAgMBAAGjUzBRMB0GA1UdDgQWBBRb40pKGU4lNvqB -1f5Mz3t0N/K3hzAfBgNVHSMEGDAWgBRb40pKGU4lNvqB1f5Mz3t0N/K3hzAPBgNV -HREECDAGhwQAAAAAAAAwCgYIKoZIzj0EAwIDSAAwRQIhAP3ST/mXyRXsU2ciRoE -gE6trllODFY+9FgT6UbF2TwzAiAAuaUxtbk6uXLqtD5NtXqOQf0Ckg8GQxc5V1G2 -9PqTXQ== ------END CERTIFICATE----- -` -} - -// Yoinked from https://cs.opensource.google/go/go/+/refs/tags/go1.21.1:src/crypto/x509/x509_test.go, line 3385 -// This allows the `buildCaCertificatePool(...)` --> `x509.ParseCertificate(...)` call error branch to be covered. -func invalidCertificateDataPEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIBBzCBrqADAgECAgEAMAoGCCqGSM49BAMCMAAwIhgPMDAwMTAxMDEwMDAwMDBa -GA8wMDAxMDEwMTAwMDAwMFowADBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABOqV -EDuVXxwZgIU3+dOwv1SsMu0xuV48hf7xmK8n7sAMYgllB+96DnPqBeboJj4snYnx -0AcE0PDVQ1l4Z3YXsQWjFTATMBEGA1UdEQEB/wQHMAWCA2FzZDAKBggqhkjOPQQD -AwNIADBFAiBi1jz/T2HT5nAfrD7zsgR+68qh7Erc6Q4qlxYBOgKG4QIhAOtjIn+Q -tA+bq+55P3ntxTOVRq0nv1mwnkjwt9cQR9Fn ------END CERTIFICATE----- -` -} - -// clientCertificatePEM returns a PEM-encoded client certificate. -// Note: The certificate is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func clientCertificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIEDDCCAvSgAwIBAgIULDFXwGrTohN/PRao2rSLk9VxFdgwDQYJKoZIhvcNAQEL -BQAwXTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEjAQBgNVBAcM -CUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVyMRQwEgYDVQQLDAtEZXZlbG9wbWVu -dDAeFw0yMzA5MjkxNzA3NTRaFw0yNDA5MjgxNzA3NTRaMGQxCzAJBgNVBAYTAlVT -MRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMQ4wDAYDVQQK -DAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFuY2VzMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqNuEZ6+TcFrmzcwp8u8mzk0jPd47GKk -H9wwdkFCzGdd8KJkFQhzLyimZIWkRDYmhaxZd76jKGBpdfyivR4e4Mi5WYlpPGMI -ppM7/rMYP8yn04tkokAazbqjOTlF8NUKqGQwqAN4Z/PvoG2HyP9omGpuLWTbjKto -oGr5aPBIhzlICU3OjHn6eKaekJeAYBo3uQFYOxCjtE9hJLDOY4q7zomMJfYoeoA2 -Afwkx1Lmozp2j/esB52/HlCKVhAOzZsPzM+E9eb1Q722dUed4OuiVYSfrDzeImrA -TufzTBTMEpFHCtdBGocZ3LRd9qmcP36ZCMsJNbYnQZV3XsI4JhjjHwIDAQABo4G8 -MIG5MBMGA1UdJQQMMAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRDl4jeiE1mJDPrYmQx -g2ndkWxpYjCBggYDVR0jBHsweaFhpF8wXTELMAkGA1UEBhMCVVMxEzARBgNVBAgM -Cldhc2hpbmd0b24xEjAQBgNVBAcMCUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVy -MRQwEgYDVQQLDAtEZXZlbG9wbWVudIIUNxx2Mr+PKXiF3d2i51fb/rnWbBgwDQYJ -KoZIhvcNAQELBQADggEBAL0wS6LkFuqGDlhaTGnAXRwRDlC6uwrm8wNWppaw9Vqt -eaZGFzodcCFp9v8jjm1LsTv7gEUBnWtn27LGP4GJSpZjiq6ulJypBxo/G0OkMByK -ky4LeGY7/BQzjzHdfXEq4gwfC45ni4n54uS9uzW3x+AwLSkxPtBxSwxhtwBLo9aE -Ql4rHUoWc81mhGO5mMZBaorxZXps1f3skfP+wZX943FIMt5gz4hkxwFp3bI/FrqH -R8DLUlCzBA9+7WIFD1wi25TV+Oyq3AjT/KiVmR+umrukhnofCWe8JiVpb5iJcd2k -Rc7+bvyb5OCnJdEX08XGWmF2/OFKLrCzLH1tQxk7VNE= ------END CERTIFICATE----- -` -} - -// clientKeyPEM returns a PEM-encoded client key. -// Note: The key is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func clientKeyPEM() string { - return ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCio24Rnr5NwWub -NzCny7ybOTSM93jsYqQf3DB2QULMZ13womQVCHMvKKZkhaRENiaFrFl3vqMoYGl1 -/KK9Hh7gyLlZiWk8Ywimkzv+sxg/zKfTi2SiQBrNuqM5OUXw1QqoZDCoA3hn8++g -bYfI/2iYam4tZNuMq2igavlo8EiHOUgJTc6Mefp4pp6Ql4BgGje5AVg7EKO0T2Ek -sM5jirvOiYwl9ih6gDYB/CTHUuajOnaP96wHnb8eUIpWEA7Nmw/Mz4T15vVDvbZ1 -R53g66JVhJ+sPN4iasBO5/NMFMwSkUcK10EahxnctF32qZw/fpkIywk1tidBlXde -wjgmGOMfAgMBAAECggEAA+R2b2yFsHW3HhVhkDqDjpF9bPxFRB8OP4b1D/d64kp9 -CJPSYmB75T6LUO+T4WAMZvmbgI6q9/3quDyuJmmQop+bNAXiY2QZYmc2sd9Wbrx2 -rczxwSJYoeDcJDP3NQ7cPPB866B9ortHWmcUr15RgghWD7cQvBqkG+bDhlvt2HKg -NZmL6R0U1bVAlRMtFJiEdMHuGnPmoDU5IGc1fKjsgijLeMboUrEaXWINoEm8ii5e -/mnsfLCBmeJAsKuXxL8/1UmvWYE/ltDfYBVclKhcH2UWTZv7pdRtHnu49lkZivUB -ZvH2DHsSMjXj6+HHr6RcRGmnMDyfhJFPCjOdTjf4oQKBgQDeYLWZx22zGXgfb7md -MhdKed9GxMJHzs4jDouqrHy0w95vwMi7RXgeKpKXiCruqSEB/Trtq01f7ekh0mvJ -Ys0h4A5tkrT5BVVBs+65uF/kSF2z/CYGNRhAABO7UM+B1e3tlnjfjeb/M78IcFbT -FyBN90A/+a9JGZ4obt3ack3afwKBgQC7OncnXC9L5QCWForJWQCNO3q3OW1Gaoxe -OAnmnPSJ7NUd7xzDNE8pzBUWXysZCoRU3QNElcQfzHWtZx1iqJPk3ERK2awNsnV7 -X2Fu4vHzIr5ZqVnM8NG7+iWrxRLf+ctcEvPiqRYo+g+r5tTGJqWh2nh9W7iQwwwE -1ikoxFBnYQKBgCbDdOR5fwXZSrcwIorkUGsLE4Cii7s4sXYq8u2tY4+fFQcl89ex -JF8dzK/dbJ5tnPNb0Qnc8n/mWN0scN2J+3gMNnejOyitZU8urk5xdUW115+oNHig -iLmfSdE9JO7c+7yOnkNZ2QpjWsl9y6TAQ0FT+D8upv93F7q0mLebdTbBAoGBALmp -r5EThD9RlvQ+5F/oZ3imO/nH88n5TLr9/St4B7NibLAjdrVIgRwkqeCmfRl26WUy -SdRQY81YtnU/JM+59fbkSsCi/FAU4RV3ryoD2QRPNs249zkYshMjawncAuyiS/xB -OyJQpI3782B3JhZdKrDG8eb19p9vG9MMAILRsh3hAoGASCvmq10nHHGFYTerIllQ -sohNaw3KDlQTkpyOAztS4jOXwvppMXbYuCznuJbHz0NEM2ww+SiA1RTvD/gosYYC -mMgqRga/Qu3b149M3wigDjK+RAcyuNGZN98bqU/UjJLjqH6IMutt59+9XNspcD96 -z/3KkMx4uqJXZyvQrmkolSg= ------END PRIVATE KEY----- -` -} - -func buildClientCertificateEntry(keyPEM, certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - certification.CertificateKeyKey: core.SecretBytes([]byte(keyPEM)), - } -} - -func buildCaCertificateEntry(certificatePEM string) map[string]core.SecretBytes { - return map[string]core.SecretBytes{ - certification.CertificateKey: core.SecretBytes([]byte(certificatePEM)), - } -} diff --git a/internal/certification/certificates.go b/internal/certification/certificates.go deleted file mode 100644 index 86b9320..0000000 --- a/internal/certification/certificates.go +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - * - * Establishes a Watcher for the Kubernetes Secrets that contain the various certificates - * and keys used to generate a tls.Config object; - * exposes the certificates and keys. - */ - -package certification - -import ( - "context" - "fmt" - "log/slog" - "sync" - - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/core" -) - -const ( - // SecretsNamespace is the value used to filter the Secrets Resource in the Informer. - SecretsNamespace = "nlk" - - // CertificateKey is the key for the certificate in the Secret. - CertificateKey = "tls.crt" - - // CertificateKeyKey is the key for the certificate key in the Secret. - CertificateKeyKey = "tls.key" -) - -type Certificates struct { - mu sync.Mutex // guards Certificates - certificates map[string]map[string]core.SecretBytes - - // CaCertificateSecretKey is the name of the Secret that contains the Certificate Authority certificate. - CaCertificateSecretKey string - - // ClientCertificateSecretKey is the name of the Secret that contains the Client certificate. - ClientCertificateSecretKey string - - // informer is the SharedInformer used to watch for changes to the Secrets . - informer cache.SharedInformer - - // K8sClient is the Kubernetes client used to communicate with the Kubernetes API. - k8sClient kubernetes.Interface - - // eventHandlerRegistration is the object used to track the event handlers with the SharedInformer. - eventHandlerRegistration cache.ResourceEventHandlerRegistration -} - -// NewCertificates factory method that returns a new Certificates object. -func NewCertificates( - k8sClient kubernetes.Interface, certificates map[string]map[string]core.SecretBytes, -) *Certificates { - return &Certificates{ - k8sClient: k8sClient, - certificates: certificates, - } -} - -// GetCACertificate returns the Certificate Authority certificate. -func (c *Certificates) GetCACertificate() core.SecretBytes { - c.mu.Lock() - defer c.mu.Unlock() - - bytes := c.certificates[c.CaCertificateSecretKey][CertificateKey] - - return bytes -} - -// GetClientCertificate returns the Client certificate and key. -func (c *Certificates) GetClientCertificate() (core.SecretBytes, core.SecretBytes) { - c.mu.Lock() - defer c.mu.Unlock() - - keyBytes := c.certificates[c.ClientCertificateSecretKey][CertificateKeyKey] - certificateBytes := c.certificates[c.ClientCertificateSecretKey][CertificateKey] - - return keyBytes, certificateBytes -} - -// Initialize initializes the Certificates object. Sets up a SharedInformer for the Secrets Resource. -func (c *Certificates) Initialize() error { - slog.Info("Certificates::Initialize") - - var err error - - c.mu.Lock() - c.certificates = make(map[string]map[string]core.SecretBytes) - c.mu.Unlock() - - informer := c.buildInformer() - - c.informer = informer - - err = c.initializeEventHandlers() - if err != nil { - return fmt.Errorf(`error occurred initializing event handlers: %w`, err) - } - - return nil -} - -// Run starts the SharedInformer. -func (c *Certificates) Run(ctx context.Context) error { - slog.Info("Certificates::Run") - - if c.informer == nil { - return fmt.Errorf(`initialize must be called before Run`) - } - - c.informer.Run(ctx.Done()) - - <-ctx.Done() - - return nil -} - -func (c *Certificates) buildInformer() cache.SharedInformer { - slog.Debug("Certificates::buildInformer") - - options := informers.WithNamespace(SecretsNamespace) - factory := informers.NewSharedInformerFactoryWithOptions(c.k8sClient, 0, options) - informer := factory.Core().V1().Secrets().Informer() - - return informer -} - -func (c *Certificates) initializeEventHandlers() error { - slog.Debug("Certificates::initializeEventHandlers") - - var err error - - handlers := cache.ResourceEventHandlerFuncs{ - AddFunc: c.handleAddEvent, - DeleteFunc: c.handleDeleteEvent, - UpdateFunc: c.handleUpdateEvent, - } - - c.eventHandlerRegistration, err = c.informer.AddEventHandler(handlers) - if err != nil { - return fmt.Errorf(`error occurred registering event handlers: %w`, err) - } - - return nil -} - -func (c *Certificates) handleAddEvent(obj interface{}) { - slog.Debug("Certificates::handleAddEvent") - - secret, ok := obj.(*corev1.Secret) - if !ok { - slog.Error("Certificates::handleAddEvent: unable to cast object to Secret") - return - } - - c.mu.Lock() - defer c.mu.Unlock() - - c.certificates[secret.Name] = map[string]core.SecretBytes{} - - // Input from the secret comes in the form - // tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVCVEN... - // tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0l... - // Where the keys are `tls.crt` and `tls.key` and the values are []byte - for k, v := range secret.Data { - c.certificates[secret.Name][k] = core.SecretBytes(v) - } - - slog.Debug("Certificates::handleAddEvent", slog.Int("certCount", len(c.certificates))) -} - -func (c *Certificates) handleDeleteEvent(obj interface{}) { - slog.Debug("Certificates::handleDeleteEvent") - - secret, ok := obj.(*corev1.Secret) - if !ok { - slog.Error("Certificates::handleDeleteEvent: unable to cast object to Secret") - return - } - - c.mu.Lock() - defer c.mu.Unlock() - - if c.certificates[secret.Name] != nil { - delete(c.certificates, secret.Name) - } - - slog.Debug("Certificates::handleDeleteEvent", slog.Int("certCount", len(c.certificates))) -} - -func (c *Certificates) handleUpdateEvent(_ interface{}, newValue interface{}) { - slog.Debug("Certificates::handleUpdateEvent") - - secret, ok := newValue.(*corev1.Secret) - if !ok { - slog.Error("Certificates::handleUpdateEvent: unable to cast object to Secret") - return - } - - c.mu.Lock() - defer c.mu.Unlock() - - for k, v := range secret.Data { - c.certificates[secret.Name][k] = v - } - - slog.Debug("Certificates::handleUpdateEvent", slog.Int("certCount", len(c.certificates))) -} diff --git a/internal/certification/certificates_test.go b/internal/certification/certificates_test.go deleted file mode 100644 index 7d2363b..0000000 --- a/internal/certification/certificates_test.go +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package certification - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" -) - -const ( - CaCertificateSecretKey = "nlk-tls-ca-secret" -) - -func TestNewCertificate(t *testing.T) { - t.Parallel() - - certificates := NewCertificates(nil, nil) - - if certificates == nil { - t.Fatalf(`certificates should not be nil`) - } -} - -func TestCertificates_Initialize(t *testing.T) { - t.Parallel() - certificates := NewCertificates(nil, nil) - - err := certificates.Initialize() - if err != nil { - t.Fatalf(`Unexpected error: %v`, err) - } -} - -func TestCertificates_RunWithoutInitialize(t *testing.T) { - t.Parallel() - certificates := NewCertificates(nil, nil) - - err := certificates.Run(context.Background()) - if err == nil { - t.Fatalf(`Expected error`) - } - - if err.Error() != `initialize must be called before Run` { - t.Fatalf(`Unexpected error: %v`, err) - } -} - -func TestCertificates_EmptyCertificates(t *testing.T) { - t.Parallel() - certificates := NewCertificates(nil, nil) - - err := certificates.Initialize() - if err != nil { - t.Fatalf(`error Initializing Certificates: %v`, err) - } - - caBytes := certificates.GetCACertificate() - if caBytes != nil { - t.Fatalf(`Expected nil CA certificate`) - } - - clientKey, clientCert := certificates.GetClientCertificate() - if clientKey != nil { - t.Fatalf(`Expected nil client key`) - } - if clientCert != nil { - t.Fatalf(`Expected nil client certificate`) - } -} - -func TestCertificates_ExerciseHandlers(t *testing.T) { - t.Parallel() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - k8sClient := fake.NewSimpleClientset() - - certificates := NewCertificates(k8sClient, nil) - - _ = certificates.Initialize() - - certificates.CaCertificateSecretKey = CaCertificateSecretKey - - //nolint:govet,staticcheck - go func() { - err := certificates.Run(context.Background()) - assert.NoError(t, err, "expected no error running certificates") - }() - - cache.WaitForCacheSync(ctx.Done(), certificates.informer.HasSynced) - - secret := buildSecret() - - /* -- Test Create -- */ - - created, err := k8sClient.CoreV1().Secrets(SecretsNamespace).Create(ctx, secret, metav1.CreateOptions{}) - if err != nil { - t.Fatalf(`error creating the Secret: %v`, err) - } - - if created.Name != secret.Name { - t.Fatalf(`Expected name %v, got %v`, secret.Name, created.Name) - } - - time.Sleep(2 * time.Second) - - caBytes := certificates.GetCACertificate() - if caBytes == nil { - t.Fatalf(`Expected non-nil CA certificate`) - } - - /* -- Test Update -- */ - - secret.Labels = map[string]string{"updated": "true"} - _, err = k8sClient.CoreV1().Secrets(SecretsNamespace).Update(ctx, secret, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf(`error updating the Secret: %v`, err) - } - - time.Sleep(2 * time.Second) - - caBytes = certificates.GetCACertificate() - if caBytes == nil { - t.Fatalf(`Expected non-nil CA certificate`) - } - - /* -- Test Delete -- */ - - err = k8sClient.CoreV1().Secrets(SecretsNamespace).Delete(ctx, secret.Name, metav1.DeleteOptions{}) - if err != nil { - t.Fatalf(`error deleting the Secret: %v`, err) - } - - time.Sleep(2 * time.Second) - - caBytes = certificates.GetCACertificate() - if caBytes != nil { - t.Fatalf(`Expected nil CA certificate, got: %v`, caBytes) - } -} - -func buildSecret() *corev1.Secret { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: CaCertificateSecretKey, - Namespace: SecretsNamespace, - }, - Data: map[string][]byte{ - CertificateKey: []byte(certificatePEM()), - CertificateKeyKey: []byte(keyPEM()), - }, - Type: corev1.SecretTypeTLS, - } -} - -// certificatePEM returns a PEM-encoded client certificate. -// Note: The certificate is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func certificatePEM() string { - return ` ------BEGIN CERTIFICATE----- -MIIEDDCCAvSgAwIBAgIULDFXwGrTohN/PRao2rSLk9VxFdgwDQYJKoZIhvcNAQEL -BQAwXTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEjAQBgNVBAcM -CUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVyMRQwEgYDVQQLDAtEZXZlbG9wbWVu -dDAeFw0yMzA5MjkxNzA3NTRaFw0yNDA5MjgxNzA3NTRaMGQxCzAJBgNVBAYTAlVT -MRMwEQYDVQQIDApXYXNoaW5ndG9uMRAwDgYDVQQHDAdTZWF0dGxlMQ4wDAYDVQQK -DAVOR0lOWDEeMBwGA1UECwwVQ29tbXVuaXR5ICYgQWxsaWFuY2VzMIIBIjANBgkq -hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqNuEZ6+TcFrmzcwp8u8mzk0jPd47GKk -H9wwdkFCzGdd8KJkFQhzLyimZIWkRDYmhaxZd76jKGBpdfyivR4e4Mi5WYlpPGMI -ppM7/rMYP8yn04tkokAazbqjOTlF8NUKqGQwqAN4Z/PvoG2HyP9omGpuLWTbjKto -oGr5aPBIhzlICU3OjHn6eKaekJeAYBo3uQFYOxCjtE9hJLDOY4q7zomMJfYoeoA2 -Afwkx1Lmozp2j/esB52/HlCKVhAOzZsPzM+E9eb1Q722dUed4OuiVYSfrDzeImrA -TufzTBTMEpFHCtdBGocZ3LRd9qmcP36ZCMsJNbYnQZV3XsI4JhjjHwIDAQABo4G8 -MIG5MBMGA1UdJQQMMAoGCCsGAQUFBwMCMB0GA1UdDgQWBBRDl4jeiE1mJDPrYmQx -g2ndkWxpYjCBggYDVR0jBHsweaFhpF8wXTELMAkGA1UEBhMCVVMxEzARBgNVBAgM -Cldhc2hpbmd0b24xEjAQBgNVBAcMCUluZGlhbm9sYTEPMA0GA1UECgwGV2FnbmVy -MRQwEgYDVQQLDAtEZXZlbG9wbWVudIIUNxx2Mr+PKXiF3d2i51fb/rnWbBgwDQYJ -KoZIhvcNAQELBQADggEBAL0wS6LkFuqGDlhaTGnAXRwRDlC6uwrm8wNWppaw9Vqt -eaZGFzodcCFp9v8jjm1LsTv7gEUBnWtn27LGP4GJSpZjiq6ulJypBxo/G0OkMByK -ky4LeGY7/BQzjzHdfXEq4gwfC45ni4n54uS9uzW3x+AwLSkxPtBxSwxhtwBLo9aE -Ql4rHUoWc81mhGO5mMZBaorxZXps1f3skfP+wZX943FIMt5gz4hkxwFp3bI/FrqH -R8DLUlCzBA9+7WIFD1wi25TV+Oyq3AjT/KiVmR+umrukhnofCWe8JiVpb5iJcd2k -Rc7+bvyb5OCnJdEX08XGWmF2/OFKLrCzLH1tQxk7VNE= ------END CERTIFICATE----- -` -} - -// keyPEM returns a PEM-encoded client key. -// Note: The key is self-signed and generated explicitly for tests, -// it is not used anywhere else. -func keyPEM() string { - return ` ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCio24Rnr5NwWub -NzCny7ybOTSM93jsYqQf3DB2QULMZ13womQVCHMvKKZkhaRENiaFrFl3vqMoYGl1 -/KK9Hh7gyLlZiWk8Ywimkzv+sxg/zKfTi2SiQBrNuqM5OUXw1QqoZDCoA3hn8++g -bYfI/2iYam4tZNuMq2igavlo8EiHOUgJTc6Mefp4pp6Ql4BgGje5AVg7EKO0T2Ek -sM5jirvOiYwl9ih6gDYB/CTHUuajOnaP96wHnb8eUIpWEA7Nmw/Mz4T15vVDvbZ1 -R53g66JVhJ+sPN4iasBO5/NMFMwSkUcK10EahxnctF32qZw/fpkIywk1tidBlXde -wjgmGOMfAgMBAAECggEAA+R2b2yFsHW3HhVhkDqDjpF9bPxFRB8OP4b1D/d64kp9 -CJPSYmB75T6LUO+T4WAMZvmbgI6q9/3quDyuJmmQop+bNAXiY2QZYmc2sd9Wbrx2 -rczxwSJYoeDcJDP3NQ7cPPB866B9ortHWmcUr15RgghWD7cQvBqkG+bDhlvt2HKg -NZmL6R0U1bVAlRMtFJiEdMHuGnPmoDU5IGc1fKjsgijLeMboUrEaXWINoEm8ii5e -/mnsfLCBmeJAsKuXxL8/1UmvWYE/ltDfYBVclKhcH2UWTZv7pdRtHnu49lkZivUB -ZvH2DHsSMjXj6+HHr6RcRGmnMDyfhJFPCjOdTjf4oQKBgQDeYLWZx22zGXgfb7md -MhdKed9GxMJHzs4jDouqrHy0w95vwMi7RXgeKpKXiCruqSEB/Trtq01f7ekh0mvJ -Ys0h4A5tkrT5BVVBs+65uF/kSF2z/CYGNRhAABO7UM+B1e3tlnjfjeb/M78IcFbT -FyBN90A/+a9JGZ4obt3ack3afwKBgQC7OncnXC9L5QCWForJWQCNO3q3OW1Gaoxe -OAnmnPSJ7NUd7xzDNE8pzBUWXysZCoRU3QNElcQfzHWtZx1iqJPk3ERK2awNsnV7 -X2Fu4vHzIr5ZqVnM8NG7+iWrxRLf+ctcEvPiqRYo+g+r5tTGJqWh2nh9W7iQwwwE -1ikoxFBnYQKBgCbDdOR5fwXZSrcwIorkUGsLE4Cii7s4sXYq8u2tY4+fFQcl89ex -JF8dzK/dbJ5tnPNb0Qnc8n/mWN0scN2J+3gMNnejOyitZU8urk5xdUW115+oNHig -iLmfSdE9JO7c+7yOnkNZ2QpjWsl9y6TAQ0FT+D8upv93F7q0mLebdTbBAoGBALmp -r5EThD9RlvQ+5F/oZ3imO/nH88n5TLr9/St4B7NibLAjdrVIgRwkqeCmfRl26WUy -SdRQY81YtnU/JM+59fbkSsCi/FAU4RV3ryoD2QRPNs249zkYshMjawncAuyiS/xB -OyJQpI3782B3JhZdKrDG8eb19p9vG9MMAILRsh3hAoGASCvmq10nHHGFYTerIllQ -sohNaw3KDlQTkpyOAztS4jOXwvppMXbYuCznuJbHz0NEM2ww+SiA1RTvD/gosYYC -mMgqRga/Qu3b149M3wigDjK+RAcyuNGZN98bqU/UjJLjqH6IMutt59+9XNspcD96 -z/3KkMx4uqJXZyvQrmkolSg= ------END PRIVATE KEY----- -` -} diff --git a/internal/certification/doc.go b/internal/certification/doc.go deleted file mode 100644 index 3388ea0..0000000 --- a/internal/certification/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -/* -Package certification includes functionality to access the Secrets containing the TLS Certificates. -*/ - -package certification diff --git a/internal/communication/factory.go b/internal/communication/factory.go index cf1bfcb..2084d5c 100644 --- a/internal/communication/factory.go +++ b/internal/communication/factory.go @@ -6,24 +6,19 @@ package communication import ( - "crypto/tls" "fmt" - "log/slog" netHttp "net/http" "time" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/authentication" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/nginxinc/kubernetes-nginx-ingress/pkg/buildinfo" ) -// NewHTTPClient is a factory method to create a new Http Client with a default configuration. -// RoundTripper is a wrapper around the default net/communication Transport to add additional headers, in this case, -// the Headers are configured for JSON. -func NewHTTPClient(settings configuration.Settings) (*netHttp.Client, error) { - headers := NewHeaders(settings.APIKey) - tlsConfig := NewTLSConfig(settings) - transport := NewTransport(tlsConfig) +// NewHTTPClient is a factory method to create a new Http Client configured for +// working with NGINXaaS or the N+ api. If skipVerify is set to true, the http +// transport will skip TLS certificate verification. +func NewHTTPClient(apiKey string, skipVerify bool) (*netHttp.Client, error) { + headers := NewHeaders(apiKey) + transport := NewTransport(skipVerify) roundTripper := NewRoundTripper(headers, transport) return &netHttp.Client{ @@ -49,22 +44,10 @@ func NewHeaders(apiKey string) []string { return headers } -// NewTLSConfig is a factory method to create a new basic Tls Config. -// More attention should be given to the use of `InsecureSkipVerify: true`, as it is not recommended for production use. -func NewTLSConfig(settings configuration.Settings) *tls.Config { - tlsConfig, err := authentication.NewTLSConfig(settings) - if err != nil { - slog.Warn("Failed to create TLS config", "error", err) - return &tls.Config{InsecureSkipVerify: true} //nolint:gosec - } - - return tlsConfig -} - // NewTransport is a factory method to create a new basic Http Transport. -func NewTransport(config *tls.Config) *netHttp.Transport { +func NewTransport(skipVerify bool) *netHttp.Transport { transport := netHttp.DefaultTransport.(*netHttp.Transport).Clone() - transport.TLSClientConfig = config + transport.TLSClientConfig.InsecureSkipVerify = skipVerify return transport } diff --git a/internal/communication/factory_test.go b/internal/communication/factory_test.go index 7562484..8c637a8 100644 --- a/internal/communication/factory_test.go +++ b/internal/communication/factory_test.go @@ -7,12 +7,14 @@ package communication import ( "testing" + + "github.com/stretchr/testify/require" ) func TestNewHTTPClient(t *testing.T) { t.Parallel() - client, err := NewHTTPClient(defaultSettings()) + client, err := NewHTTPClient("fakeKey", true) if err != nil { t.Fatalf(`Unexpected error: %v`, err) } @@ -80,8 +82,7 @@ func TestNewHeadersWithNoAPIKey(t *testing.T) { func TestNewTransport(t *testing.T) { t.Parallel() - config := NewTLSConfig(defaultSettings()) - transport := NewTransport(config) + transport := NewTransport(false) if transport == nil { t.Fatalf(`transport should not be nil`) @@ -91,11 +92,5 @@ func TestNewTransport(t *testing.T) { t.Fatalf(`transport.TLSClientConfig should not be nil`) } - if transport.TLSClientConfig != config { - t.Fatalf(`transport.TLSClientConfig should be the same as config`) - } - - if !transport.TLSClientConfig.InsecureSkipVerify { - t.Fatalf(`transport.TLSClientConfig.InsecureSkipVerify should be true`) - } + require.False(t, transport.TLSClientConfig.InsecureSkipVerify) } diff --git a/internal/communication/roundtripper_test.go b/internal/communication/roundtripper_test.go index 55dea88..d00af17 100644 --- a/internal/communication/roundtripper_test.go +++ b/internal/communication/roundtripper_test.go @@ -9,15 +9,13 @@ import ( "bytes" netHttp "net/http" "testing" - - "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" ) func TestNewRoundTripper(t *testing.T) { t.Parallel() headers := NewHeaders("fakeKey") - transport := NewTransport(NewTLSConfig(defaultSettings())) + transport := NewTransport(true) roundTripper := NewRoundTripper(headers, transport) if roundTripper == nil { @@ -57,7 +55,7 @@ func TestRoundTripperRoundTrip(t *testing.T) { t.Parallel() headers := NewHeaders("fakeKey") - transport := NewTransport(NewTLSConfig(defaultSettings())) + transport := NewTransport(true) roundTripper := NewRoundTripper(headers, transport) request, err := NewRequest("GET", "http://example.com", nil) @@ -92,9 +90,3 @@ func NewRequest(method string, url string, body []byte) (*netHttp.Request, error return request, nil } - -func defaultSettings() configuration.Settings { - return configuration.Settings{ - TLSMode: configuration.NoTLS, - } -} diff --git a/internal/configuration/configuration_test.go b/internal/configuration/configuration_test.go index fc41974..45764d6 100644 --- a/internal/configuration/configuration_test.go +++ b/internal/configuration/configuration_test.go @@ -4,13 +4,12 @@ import ( "testing" "time" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" "github.com/nginxinc/kubernetes-nginx-ingress/internal/configuration" "github.com/stretchr/testify/require" ) -func TestConfiguration(t *testing.T) { +func TestConfiguration_Read(t *testing.T) { t.Parallel() tests := map[string]struct { @@ -22,11 +21,7 @@ func TestConfiguration(t *testing.T) { expectedSettings: configuration.Settings{ LogLevel: "warn", NginxPlusHosts: []string{"https://10.0.0.1:9000/api"}, - TLSMode: configuration.NoTLS, - Certificates: &certification.Certificates{ - CaCertificateSecretKey: "fakeCAKey", - ClientCertificateSecretKey: "fakeCertKey", - }, + SkipVerifyTLS: false, Handler: configuration.HandlerSettings{ RetryCount: 5, Threads: 1, @@ -58,11 +53,7 @@ func TestConfiguration(t *testing.T) { expectedSettings: configuration.Settings{ LogLevel: "warn", NginxPlusHosts: []string{"https://10.0.0.1:9000/api", "https://10.0.0.2:9000/api"}, - TLSMode: configuration.NoTLS, - Certificates: &certification.Certificates{ - CaCertificateSecretKey: "fakeCAKey", - ClientCertificateSecretKey: "fakeCertKey", - }, + SkipVerifyTLS: true, Handler: configuration.HandlerSettings{ RetryCount: 5, Threads: 1, @@ -100,3 +91,48 @@ func TestConfiguration(t *testing.T) { }) } } + +func TestConfiguration_TLS(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + tlsMode string + expectedSkipVerifyTLS bool + expectedErr bool + }{ + "no input": { + tlsMode: "", + expectedSkipVerifyTLS: false, + }, + "no tls": { + tlsMode: "no-tls", + expectedSkipVerifyTLS: true, + }, + "skip verify tls": { + tlsMode: "skip-verify-tls", + expectedSkipVerifyTLS: true, + }, + "ca tls": { + tlsMode: "ca-tls", + expectedSkipVerifyTLS: false, + }, + "unexpected input": { + tlsMode: "unexpected-tls-mode", + expectedErr: true, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + skipVerifyTLS, err := configuration.ValidateTLSMode(tc.tlsMode) + if tc.expectedErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + require.Equal(t, tc.expectedSkipVerifyTLS, skipVerifyTLS) + }) + } +} diff --git a/internal/configuration/settings.go b/internal/configuration/settings.go index ebeacc4..75cec2e 100644 --- a/internal/configuration/settings.go +++ b/internal/configuration/settings.go @@ -11,8 +11,6 @@ import ( "log/slog" "time" - "github.com/nginxinc/kubernetes-nginx-ingress/internal/certification" - "github.com/spf13/viper" ) @@ -110,16 +108,12 @@ type Settings struct { // NginxPlusHosts is a list of Nginx Plus hosts that will be used to update the Border Servers. NginxPlusHosts []string - // TlsMode is the value used to determine which of the five TLS modes will be used to communicate - // with the Border Servers (see: ../../docs/tls/README.md). - TLSMode TLSMode + // SkipVerifyTLS determines whether the http client will skip TLS verification or not. + SkipVerifyTLS bool // APIKey is the api key used to authenticate with the dataplane API. APIKey string - // Certificates is the object used to retrieve the certificates and keys used to communicate with the Border Servers. - Certificates *certification.Certificates - // Handler contains the configuration values needed by the Handler. Handler HandlerSettings @@ -144,11 +138,13 @@ func Read(configName, configPath string) (s Settings, err error) { return s, err } - tlsMode := NoTLS - if t, err := validateTLSMode(v.GetString("tls-mode")); err != nil { + skipVerifyTLS, err := ValidateTLSMode(v.GetString("tls-mode")) + if err != nil { slog.Error("could not validate tls mode", "error", err) - } else { - tlsMode = t + } + + if skipVerifyTLS { + slog.Warn("skipping TLS verification for NGINX hosts") } serviceAnnotation := DefaultServiceAnnotation @@ -159,12 +155,8 @@ func Read(configName, configPath string) (s Settings, err error) { return Settings{ LogLevel: v.GetString("log-level"), NginxPlusHosts: v.GetStringSlice("nginx-hosts"), - TLSMode: tlsMode, + SkipVerifyTLS: skipVerifyTLS, APIKey: base64.StdEncoding.EncodeToString([]byte(v.GetString("NGINXAAS_DATAPLANE_API_KEY"))), - Certificates: &certification.Certificates{ - CaCertificateSecretKey: v.GetString("ca-certificate"), - ClientCertificateSecretKey: v.GetString("client-certificate"), - }, Handler: HandlerSettings{ RetryCount: 5, Threads: 1, @@ -192,10 +184,15 @@ func Read(configName, configPath string) (s Settings, err error) { }, nil } -func validateTLSMode(tlsConfigMode string) (TLSMode, error) { - if tlsMode, tlsModeFound := TLSModeMap[tlsConfigMode]; tlsModeFound { - return tlsMode, nil +func ValidateTLSMode(tlsConfigMode string) (skipVerify bool, err error) { + if tlsConfigMode == "" { + return false, nil + } + + var tlsModeFound bool + if skipVerify, tlsModeFound = tlsModeMap[tlsConfigMode]; tlsModeFound { + return skipVerify, nil } - return NoTLS, fmt.Errorf(`invalid tls-mode value: %s`, tlsConfigMode) + return false, fmt.Errorf(`invalid tls-mode value: %s`, tlsConfigMode) } diff --git a/internal/configuration/testdata/one-nginx-host.yaml b/internal/configuration/testdata/one-nginx-host.yaml index f05d81e..45e55ef 100644 --- a/internal/configuration/testdata/one-nginx-host.yaml +++ b/internal/configuration/testdata/one-nginx-host.yaml @@ -2,7 +2,6 @@ ca-certificate: "fakeCAKey" client-certificate: "fakeCertKey" log-level: "warn" nginx-hosts: "https://10.0.0.1:9000/api" -tls-mode: "no-tls" service-annotation-match: "fakeServiceMatch" creationTimestamp: "2024-09-04T17:59:20Z" name: "nlk-config" diff --git a/internal/configuration/tlsmodes.go b/internal/configuration/tlsmodes.go index 2f7271f..e329047 100644 --- a/internal/configuration/tlsmodes.go +++ b/internal/configuration/tlsmodes.go @@ -6,41 +6,19 @@ package configuration const ( - NoTLS TLSMode = iota - CertificateAuthorityTLS - CertificateAuthorityMutualTLS - SelfSignedTLS - SelfSignedMutualTLS + // NoTLS is deprecated as misleading. It is the same as SkipVerifyTLS. + NoTLS = "no-tls" + // SkipVerifyTLS causes the http client to skip verification of the NGINX + // host's certificate chain and host name. + SkipVerifyTLS = "skip-verify-tls" + // CertificateAuthorityTLS is deprecated as misleading. This is the same as + // the default behavior which is to verify the NGINX hosts's certificate + // chain and host name, if https is used. + CertificateAuthorityTLS = "ca-tls" ) -const ( - NoTLSString = "no-tls" - CertificateAuthorityTLSString = "ca-tls" - CertificateAuthorityMutualTLSString = "ca-mtls" - SelfSignedTLSString = "ss-tls" - SelfSignedMutualTLSString = "ss-mtls" -) - -type TLSMode int - -var TLSModeMap = map[string]TLSMode{ - NoTLSString: NoTLS, - CertificateAuthorityTLSString: CertificateAuthorityTLS, - CertificateAuthorityMutualTLSString: CertificateAuthorityMutualTLS, - SelfSignedTLSString: SelfSignedTLS, - SelfSignedMutualTLSString: SelfSignedMutualTLS, -} - -func (t TLSMode) String() string { - modes := []string{ - NoTLSString, - CertificateAuthorityTLSString, - CertificateAuthorityMutualTLSString, - SelfSignedTLSString, - SelfSignedMutualTLSString, - } - if t < NoTLS || t > SelfSignedMutualTLS { - return "" - } - return modes[t] +var tlsModeMap = map[string]bool{ + NoTLS: true, + SkipVerifyTLS: true, + CertificateAuthorityTLS: false, } diff --git a/internal/configuration/tlsmodes_test.go b/internal/configuration/tlsmodes_test.go deleted file mode 100644 index d849cd9..0000000 --- a/internal/configuration/tlsmodes_test.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package configuration - -import ( - "testing" -) - -func Test_String(t *testing.T) { - t.Parallel() - mode := NoTLS.String() - if mode != "no-tls" { - t.Errorf("Expected TLSModeNoTLS to be 'no-tls', got '%s'", mode) - } - - mode = CertificateAuthorityTLS.String() - if mode != "ca-tls" { - t.Errorf("Expected TLSModeCaTLS to be 'ca-tls', got '%s'", mode) - } - - mode = CertificateAuthorityMutualTLS.String() - if mode != "ca-mtls" { - t.Errorf("Expected TLSModeCaMTLS to be 'ca-mtls', got '%s'", mode) - } - - mode = SelfSignedTLS.String() - if mode != "ss-tls" { - t.Errorf("Expected TLSModeSsTLS to be 'ss-tls', got '%s'", mode) - } - - mode = SelfSignedMutualTLS.String() - if mode != "ss-mtls" { - t.Errorf("Expected TLSModeSsMTLS to be 'ss-mtls', got '%s',", mode) - } - - mode = TLSMode(5).String() - if mode != "" { - t.Errorf("Expected TLSMode(5) to be '', got '%s'", mode) - } -} - -func Test_TLSModeMap(t *testing.T) { - t.Parallel() - mode := TLSModeMap["no-tls"] - if mode != NoTLS { - t.Errorf("Expected TLSModeMap['no-tls'] to be TLSModeNoTLS, got '%d'", mode) - } - - mode = TLSModeMap["ca-tls"] - if mode != CertificateAuthorityTLS { - t.Errorf("Expected TLSModeMap['ca-tls'] to be TLSModeCaTLS, got '%d'", mode) - } - - mode = TLSModeMap["ca-mtls"] - if mode != CertificateAuthorityMutualTLS { - t.Errorf("Expected TLSModeMap['ca-mtls'] to be TLSModeCaMTLS, got '%d'", mode) - } - - mode = TLSModeMap["ss-tls"] - if mode != SelfSignedTLS { - t.Errorf("Expected TLSModeMap['ss-tls'] to be TLSModeSsTLS, got '%d'", mode) - } - - mode = TLSModeMap["ss-mtls"] - if mode != SelfSignedMutualTLS { - t.Errorf("Expected TLSModeMap['ss-mtls'] to be TLSModeSsMTLS, got '%d'", mode) - } - - mode = TLSModeMap["invalid"] - if mode != TLSMode(0) { - t.Errorf("Expected TLSModeMap['invalid'] to be TLSMode(0), got '%d'", mode) - } -} diff --git a/internal/core/secret_bytes.go b/internal/core/secret_bytes.go deleted file mode 100644 index 0bbc3bf..0000000 --- a/internal/core/secret_bytes.go +++ /dev/null @@ -1,21 +0,0 @@ -package core - -import ( - "encoding/json" -) - -// Wraps byte slices which potentially could contain -// sensitive data that should not be output to the logs. -// This will output [REDACTED] if attempts are made -// to print this type in logs, serialize to JSON, or -// otherwise convert it to a string. -// Usage: core.SecretBytes(myByteSlice) -type SecretBytes []byte - -func (sb SecretBytes) String() string { - return "[REDACTED]" -} - -func (sb SecretBytes) MarshalJSON() ([]byte, error) { - return json.Marshal("[REDACTED]") -} diff --git a/internal/core/secret_bytes_test.go b/internal/core/secret_bytes_test.go deleted file mode 100644 index 5e6bc3f..0000000 --- a/internal/core/secret_bytes_test.go +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 F5 Inc. All rights reserved. - * Use of this source code is governed by the Apache License that can be found in the LICENSE file. - */ - -package core - -import ( - "encoding/json" - "fmt" - "testing" -) - -func TestSecretBytesToString(t *testing.T) { - t.Parallel() - sensitive := SecretBytes([]byte("If you can see this we have a problem")) - - expected := "foo [REDACTED] bar" - result := fmt.Sprintf("foo %v bar", sensitive) - if result != expected { - t.Errorf("Expected %s, got %s", expected, result) - } -} - -func TestSecretBytesToJSON(t *testing.T) { - t.Parallel() - sensitive, _ := json.Marshal(SecretBytes([]byte("If you can see this we have a problem"))) - expected := `foo "[REDACTED]" bar` - result := fmt.Sprintf("foo %v bar", string(sensitive)) - if result != expected { - t.Errorf("Expected %s, got %s", expected, result) - } -} diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index 80faaa3..bbae5e1 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -126,8 +126,7 @@ func (s *Synchronizer) buildBorderClient(event *core.ServerUpdateEvent) (applica slog.Debug(`Synchronizer::buildBorderClient`) var err error - - httpClient, err := communication.NewHTTPClient(s.settings) + httpClient, err := communication.NewHTTPClient(s.settings.APIKey, s.settings.SkipVerifyTLS) if err != nil { return nil, fmt.Errorf(`error creating HTTP client: %v`, err) } From 06d6bd1b398a90ca751d219978395aed78d8b0e6 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Thu, 13 Mar 2025 16:30:52 -0600 Subject: [PATCH 132/136] NLB-6295 Bumped version to 1.2.0 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 65087b4..26aaba0 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1.4 +1.2.0 From a8edd189af3a1afdb499ced9129713f02377ca03 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 1 Jul 2025 11:51:34 -0600 Subject: [PATCH 133/136] Updated go version to 1.24.4 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c09a4b4..b49ed5c 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ module github.com/nginxinc/kubernetes-nginx-ingress -go 1.24.3 +go 1.24.4 require ( github.com/nginx/nginx-plus-go-client/v2 v2.3.0 From 44894101992010b509091ebdbf57eaf965d52fed Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 2 Jul 2025 14:35:47 -0600 Subject: [PATCH 134/136] NLB-6754 When deleting upstream servers handle upstream not found error Now that the plus go client allows users to check the http status code of the error, handle the upstream not found case by doing nothing. --- go.mod | 4 ++-- go.sum | 8 ++++---- internal/synchronization/synchronizer.go | 15 ++++++++++++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index b49ed5c..6a78fe2 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ module github.com/nginxinc/kubernetes-nginx-ingress go 1.24.4 require ( - github.com/nginx/nginx-plus-go-client/v2 v2.3.0 + github.com/nginx/nginx-plus-go-client/v2 v2.4.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/sync v0.12.0 + golang.org/x/sync v0.13.0 k8s.io/api v0.33.1 k8s.io/apimachinery v0.33.1 k8s.io/client-go v0.33.1 diff --git a/go.sum b/go.sum index 80d0e9d..8cf5ed8 100644 --- a/go.sum +++ b/go.sum @@ -63,8 +63,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nginx/nginx-plus-go-client/v2 v2.3.0 h1:ciKh1lwadNzUaOGjLcKWu/BGigASxU6p7v/6US71fhA= -github.com/nginx/nginx-plus-go-client/v2 v2.3.0/go.mod h1:U7G5pqucUS1V4Uecs1xCsJ9knSsfwqhwu8ZEjoCYnmk= +github.com/nginx/nginx-plus-go-client/v2 v2.4.0 h1:4c7V57CLCZUOxQCUcS9G8a5MClzdmxByBm+f4zKMzAY= +github.com/nginx/nginx-plus-go-client/v2 v2.4.0/go.mod h1:P+dIP2oKYzFoyf/zlLWQa8Sf+fHb+CclOKzxAjxpvug= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= @@ -135,8 +135,8 @@ golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/synchronization/synchronizer.go b/internal/synchronization/synchronizer.go index bbae5e1..8f0fff6 100644 --- a/internal/synchronization/synchronizer.go +++ b/internal/synchronization/synchronizer.go @@ -10,7 +10,7 @@ import ( "errors" "fmt" "log/slog" - "strings" + "net/http" "time" nginxClient "github.com/nginx/nginx-plus-go-client/v2/client" @@ -36,6 +36,13 @@ type Interface interface { ShutDown() } +// StatusError is a wrapper for errors from the go plus client that contain http +// status codes. +type StatusError interface { + Status() int + Code() string +} + type Translator interface { Translate(*core.Event) (core.ServerUpdateEvents, error) } @@ -264,11 +271,13 @@ func (s *Synchronizer) handleDeletedEvent(ctx context.Context, serverUpdateEvent err = borderClient.Update(ctx, serverUpdateEvent) + var se StatusError switch { case err == nil: return nil - // checking the string is not ideal, but the plus client gives us no option - case strings.Contains(err.Error(), "status=404"): + case errors.As(err, &se) && se.Status() == http.StatusNotFound: + // if the user has already removed the upstream from their NGINX + // configuration there is nothing left to do return nil default: return fmt.Errorf(`error occurred deleting the %s upstream servers: %w`, serverUpdateEvent.ClientType, err) From 38bf9ab4fa68983f6f19ecaf89e5a603d5c1cba7 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Wed, 2 Jul 2025 14:57:04 -0600 Subject: [PATCH 135/136] NLB-6754 Bumped version to 1.2.1 --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 26aaba0..6085e94 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.2.0 +1.2.1 From a9c0f148d5d324d8b6dda4c0ed240caca9b07237 Mon Sep 17 00:00:00 2001 From: Alex Russell Date: Tue, 8 Jul 2025 16:27:20 -0600 Subject: [PATCH 136/136] Removed context as a field within the nginx stream border client This is a go anti-pattern --- internal/application/nginx_stream_border_client.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/application/nginx_stream_border_client.go b/internal/application/nginx_stream_border_client.go index a65be86..238a22b 100644 --- a/internal/application/nginx_stream_border_client.go +++ b/internal/application/nginx_stream_border_client.go @@ -18,7 +18,6 @@ import ( type NginxStreamBorderClient struct { BorderClient nginxClient NginxClientInterface - ctx context.Context } // NewNginxStreamBorderClient is the Factory function for creating an NginxStreamBorderClient. @@ -30,7 +29,6 @@ func NewNginxStreamBorderClient(client interface{}) (Interface, error) { return &NginxStreamBorderClient{ nginxClient: ngxClient, - ctx: context.Background(), }, nil }