From 2fc3eb00b977c6c81f803c4ad7c1e59ffac42fa9 Mon Sep 17 00:00:00 2001 From: Douglas-Lee Date: Thu, 23 Oct 2025 22:14:28 +0800 Subject: [PATCH 1/3] refactor(*): refactor configuration --- .github/workflows/release-test.yml | 2 +- .github/workflows/test.yml | 72 ++++++++----- Dockerfile | 3 +- Makefile | 15 ++- README.md | 16 +-- cmd/db.go | 7 +- cmd/root.go | 2 +- config.yml | 130 ++++++++++++------------ config/admin.go | 2 +- config/config.go | 4 +- config/proxy.go | 2 +- config/status.go | 2 +- config/worker.go | 2 +- docker-compose-clustering.yml | 10 +- docker-compose.yml | 14 +-- openapi-status.yml | 2 +- openapi.yml | 2 +- test/accesslog/accesslog_test.go | 4 - test/admin/attempts_test.go | 4 +- test/admin/debug_test.go | 1 - test/admin/endpoints_test.go | 4 +- test/admin/events_test.go | 4 +- test/admin/listen_test.go | 5 +- test/admin/metadata_test.go | 4 +- test/admin/middlewares_test.go | 8 +- test/admin/plugins_test.go | 4 +- test/admin/sources_test.go | 4 +- test/admin/workspaces_test.go | 4 +- test/cmd/admin_test.go | 20 ++-- test/declarative/declarative_test.go | 4 +- test/delivery/acl_test.go | 10 +- test/delivery/delivery_test.go | 78 ++++---------- test/delivery/http_proxy_test.go | 24 ++--- test/helper/helper.go | 33 +++++- test/metrics/opentelemetry_test.go | 3 - test/plugins/function_test.go | 6 +- test/plugins/wasm_test.go | 6 +- test/plugins/webhookx_signature_test.go | 6 +- test/proxy/ingest_test.go | 3 +- test/proxy/listen_test.go | 5 +- test/proxy/ratelimit_test.go | 4 +- test/tracing/admin_test.go | 31 +++--- test/tracing/ginkgo_test.go | 1 - test/tracing/proxy_test.go | 4 +- test/tracing/worker_test.go | 2 - 45 files changed, 250 insertions(+), 323 deletions(-) diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index d9fd84b..5c8ff6a 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -17,7 +17,7 @@ jobs: run: | docker compose up -d sleep 10 - STATUS_CODE=$(curl -o /dev/null -s -w "%{http_code}" http://localhost:8080) + STATUS_CODE=$(curl -o /dev/null -s -w "%{http_code}" http://localhost:9601) if [ "$STATUS_CODE" -ne 200 ]; then echo "API failed with status code $STATUS_CODE" exit 1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99d54b6..8f04a4b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,8 @@ name: Test +permissions: + contents: read + on: pull_request: paths-ignore: @@ -13,59 +16,78 @@ on: - '**/*.gif' jobs: - unit-tests: + tests-unit: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 with: go-version-file: go.mod - - - name: run tests - run: make test-coverage - - - name: upload coverage reselt to Codecov + - name: runs tests + run: | + make test-unit FLAGS=-coverprofile=coverage.txt + cat coverage.txt + - name: uploads coverage reselt uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true flags: unit - services: - httpbin: - image: kennethreitz/httpbin - ports: - - 9999:80 - - integration-tests: + tests-integration-main: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 with: go-version-file: go.mod - - - name: start dependencies + - name: starts dependencies run: | mkdir -p test/output/otel sudo chmod 777 -R test/output/otel make test-deps sleep 3 docker compose -f test/docker-compose.yml logs - - - name: install + - name: installs webhookx run: make install - + - name: installs ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo - name: run tests - run: make test-integration-coverage - - - name: upload coverage reselt to Codecov + run: | + make test-main FLAGS="-coverpkg=github.com/webhookx-io/webhookx/... --cover --coverprofile=coverage.txt" + cat coverage.txt + - name: upload coverage reselt uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true flags: integration + + tests-integration-o11: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: starts dependencies + run: | + mkdir -p test/output/otel + sudo chmod 777 -R test/output/otel + make test-deps + sleep 3 + docker compose -f test/docker-compose.yml logs + - name: installs webhookx + run: make install + - name: install ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo + - name: run tests + run: | + make test-o11 FLAGS="-coverpkg=github.com/webhookx-io/webhookx/... --cover --coverprofile=coverage.txt" + cat coverage.txt + - name: upload coverage reselt + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true + flags: integration-o11 diff --git a/Dockerfile b/Dockerfile index 6fc260d..0fb8b4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ FROM alpine:3.15 COPY --from=build-env /go/src/webhookx-io/webhookx/webhookx /usr/local/bin -EXPOSE 8080 +EXPOSE 9600 + CMD ["webhookx", "start"] diff --git a/Makefile b/Makefile index afa0160..f9a731e 100644 --- a/Makefile +++ b/Makefile @@ -24,17 +24,14 @@ test-deps: mkdir -p test/output/otel docker compose -f test/docker-compose.yml up -d -test: clean - go test $$(go list ./... | grep -v /test/ | grep -v /examples/ ) +test-unit: clean + go test $$(go list ./... | grep -v /test/ | grep -v /examples/ ) $(FLAGS) -test-coverage: clean - go test $$(go list ./... | grep -v /test/ | grep -v /examples/ ) -coverprofile=coverage.txt +test-o11: clean + ginkgo -r $(FLAGS) ./test/metrics ./test/tracing -test-integration: clean - go test -p 1 -v ./test/... - -test-integration-coverage: clean - go test -p 1 -v ./test/... --coverpkg ./... -coverprofile=coverage.txt +test-main: clean + ginkgo -r --skip-package=metrics,tracing $(FLAGS) ./test goreleaser: goreleaser release --snapshot --clean diff --git a/README.md b/README.md index c90d4d3..140fb39 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ WebhookX is an open-source webhooks gateway for message receiving, processing, a ## Features -- **Admin API:** Expose a RESTful API on port `:8080` for managing WebhookX entities. +- **Admin API:** Expose a RESTful API on port `:9601` for managing WebhookX entities. - **Retries:** Automatically retry unsuccessful deliveries with configurable delays. - **Fan out:** Route events to multiple endpoints based on the event type. - **Rate Limiting:** Protect the gateway ingestion and delivery endpoints from overload. @@ -67,7 +67,7 @@ curl -O https://raw.githubusercontent.com/webhookx-io/webhookx/main/docker-compo Once it's running, you will see HTTP 200 response ``` -curl http://localhost:8080 +curl http://localhost:9601 ``` ```http @@ -102,10 +102,10 @@ Once it is set up, you're ready to send events to WebhookX. ### 3. Send events to WebhookX -> The Ingestion is exposed on port `:8081` +> The Ingestion is exposed on port `:9600` ``` -curl -X POST http://localhost:8081 \ +curl -X POST http://localhost:9600 \ --header 'Content-Type: application/json' \ --data '{ "event_type": "charge.succeeded", @@ -125,12 +125,12 @@ We sent a `charge.succeeded` event including `data` to WebhookX, and it will be > > Attempt object represents the delivery result of an event, and contains inspection information. -> The Admin is exposed on port `:8080` +> The Admin is exposed on port `:9601` Let's make a request to retrieve the attempt list ``` -curl http://localhost:8080/workspaces/default/attempts +curl http://localhost:9601/workspaces/default/attempts ```
@@ -174,7 +174,7 @@ curl http://localhost:8080/workspaces/default/attempts To inspect the data such as `request.headers`, `request.body`, `response.headers`, and `response.body`, try ``` -http://localhost:8080/workspaces/default/attempts/338lax8Xe774EhimzBukip37Zne +http://localhost:9601/workspaces/default/attempts/338lax8Xe774EhimzBukip37Zne ```
@@ -243,8 +243,8 @@ Usage: Available Commands: admin Admin commands completion Generate the autocompletion script for the specified shell + db Database commands help Help about any command - migrations start Start server version Print the version diff --git a/cmd/db.go b/cmd/db.go index 2add0af..390f487 100644 --- a/cmd/db.go +++ b/cmd/db.go @@ -48,10 +48,9 @@ func newDatabaseResetCmd() *cobra.Command { func newDatabaseCmd() *cobra.Command { database := &cobra.Command{ - Use: "db", - Aliases: []string{"migrations"}, - Short: "Database commands", - Long: ``, + Use: "db", + Short: "Database commands", + Long: ``, } database.PersistentFlags().StringVarP(&configurationFile, "config", "", "", "The configuration filename") diff --git a/cmd/root.go b/cmd/root.go index e586cb2..f3162f3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -7,7 +7,7 @@ import ( ) const ( - defaultAdminURL = "http://localhost:8080" + defaultAdminURL = "http://localhost:9601" ) var ( diff --git a/config.yml b/config.yml index db5ae33..3038398 100644 --- a/config.yml +++ b/config.yml @@ -1,15 +1,14 @@ -# --------------------------- -# WebhookX configuration file -# --------------------------- - +# ------------------------------------ +# WebhookX Configuration YAML file +# ------------------------------------ log: - file: /dev/stdout - level: info # supported values are debug, info, warn, and error. - format: text # supported values are "text" and "json" + file: /dev/stdout # Specifies the log file. + level: info # Supported values: debug, info, warn, and error. + format: text # Supported values: text, json. -access_log: - file: /dev/stdout - format: text # supported values are "text" and "json" +access_log: # Access Log + file: /dev/stdout # Specifies the log file. + format: text # Supported values: text, json. database: host: localhost @@ -17,10 +16,11 @@ database: username: webhookx password: database: webhookx - parameters: 'application_name=webhookx&sslmode=disable&connect_timeout=10' # The connection uri parameters. - # See https://www.postgresql.org/docs/current/libpq-connect.html - max_pool_size: 40 # The maximum number of connections - max_lifetime: 1800 # The maximum lifetime (in seconds) of a connection + parameters: 'application_name=webhookx&sslmode=disable&connect_timeout=10' + # Connection uri parameters. + # See https://www.postgresql.org/docs/current/libpq-connect.html for more information. + max_pool_size: 40 # Specifies the maximum number of connections. + max_lifetime: 1800 # Specifies the maximum lifetime (in seconds) for a connection. redis: host: localhost @@ -29,47 +29,55 @@ redis: database: 0 #------------------------------------------------------------------------------ -# Cluster +# PROXY #------------------------------------------------------------------------------ -#role: standalone # Enables cluster mode. - # This allows some nodes in the cluster to run as the control plane - # and others to run as the data plane. - # - # supported values are: - # - # - `standalone`: disable cluster mode. - # - `cp`: this node runs as the control plane. - # It connects to the database to provide entity management. - # - `dp_proxy`: this node runs as the Proxy data plane. - # - `dp_worker`: this node runs as the Worker data plane. - -anonymous_reports: true # sends anonymous data such as software version to WebhookX. +proxy: + listen: 0.0.0.0:9600 + tls: # TLS configuration + cert: '' # The path to TLS certificate. Example: /path/to/server.cert + key: '' # The path to TLS certificate key. Example: /path/to/server.key + timeout_read: 10 # Specifies the maximum time (in seconds) for reading request. 0 disables timeout. + timeout_write: 10 # Specifies the maximum time (in seconds) for writing response. 0 disables timeout. + max_request_body_size: 1048576 # Specifies the maximum request body size. Default is 1048576. + response: # Default HTTP response + code: 200 + content_type: application/json + body: '{"message": "OK"}' + queue: # Queue settings + type: redis # Queue type. Supported values: redis, off. + redis: + host: localhost + port: 6379 + password: + database: 0 #------------------------------------------------------------------------------ -# ADMIN +# Admin API +# The Admin API provides RESTful APIs for managing entities. #------------------------------------------------------------------------------ - admin: - #listen: 127.0.0.1:8080 - #debug_endpoints: true # enables debugging and profiling endpoints. see https://pkg.go.dev/net/http/pprof - #tls: - # cert: /path/to/server.crt - # key: /path/to/server.key + listen: 127.0.0.1:9601 + debug_endpoints: false # Whether to expose debugging and profiling endpoints. + # See https://pkg.go.dev/net/http/pprof for more information. + #tls: # TLS configuration + # cert: /path/to/server.crt # The Path to TLS certificate. + # key: /path/to/server.key # The path to TLS certificate key. #------------------------------------------------------------------------------ -# STATUS +# Status API +# The Status API provides APIs for retrieving runtime status about WebhookX. #------------------------------------------------------------------------------ - status: - listen: 127.0.0.1:8082 - debug_endpoints: true # enables debugging and profiling endpoints. see https://pkg.go.dev/net/http/pprof + listen: 127.0.0.1:9602 + debug_endpoints: true # Whether to expose debugging and profiling endpoints. + # See https://pkg.go.dev/net/http/pprof for more information. #------------------------------------------------------------------------------ -# WORKER +# Worker +# The Worker sends events to destinations. #------------------------------------------------------------------------------ - worker: - enabled: false # Whether to enable the Worker. + enabled: true # Whether to enable the Worker. deliverer: timeout: 60000 # Sets the request timeout (in milliseconds) for delivery requests. acl: # Access Control List (ACL) defines rules to control outbound network access. @@ -105,28 +113,22 @@ worker: concurrency: 0 # pool concurrency, default to 100 * CPUs #------------------------------------------------------------------------------ -# PROXY +# Cluster #------------------------------------------------------------------------------ -proxy: - #listen: 127.0.0.1:8081 - #tls: - # cert: /path/to/server.crt - # key: /path/to/server.key - timeout_read: 10 # read timeout (in seconds), 0 indicates unlimited. - timeout_write: 60 # write timeout (in seconds), 0 indicates unlimited. - max_request_body_size: 1048576 - response: - code: 200 - content_type: application/json - body: '{"message": "OK"}' +#role: standalone # Enables cluster mode. + # This allows some nodes in the cluster to run as the control plane + # and others to run as the data plane. + # + # supported values are: + # + # - `standalone`: disable cluster mode. + # - `cp`: this node runs as the control plane. + # It connects to the database to provide entity management. + # - `dp_proxy`: this node runs as the Proxy data plane. + # - `dp_worker`: this node runs as the Worker data plane. + +anonymous_reports: true # sends anonymous data such as software version to WebhookX. - queue: - type: redis # supported values are redis, off - redis: - host: localhost - port: 6379 - password: - database: 0 #------------------------------------------------------------------------------ # METRICS @@ -137,7 +139,7 @@ metrics: #exports: [ opentelemetry ] # list of enabled vendor exports. supported value are opentelemetry push_interval: 10 # interval(in seconds) at which metrics are sent to the OpenTelemetry Collector opentelemetry: - protocol: http/protobuf # supported value are http/protobuf, grpc + protocol: http/protobuf # Supported values: http/protobuf, grpc. endpoint: http://localhost:4318/v1/metrics # http/protobuf(http://localhost:4318/v1/metrics), grpc(localhost:4317) #------------------------------------------------------------------------------ @@ -149,5 +151,5 @@ tracing: env: prod sampling_rate: 1.0 opentelemetry: - protocol: http/protobuf # supported value are http/protobuf, grpc + protocol: http/protobuf # Supported value: http/protobuf, grpc. endpoint: http://localhost:4318/v1/traces # http/protobuf(http://localhost:4318/v1/traces), grpc(localhost:4317) diff --git a/config/admin.go b/config/admin.go index a8ca744..2b59164 100644 --- a/config/admin.go +++ b/config/admin.go @@ -1,7 +1,7 @@ package config type AdminConfig struct { - Listen string `yaml:"listen" json:"listen"` + Listen string `yaml:"listen" json:"listen" default:"127.0.0.1:9601"` DebugEndpoints bool `yaml:"debug_endpoints" json:"debug_endpoints" envconfig:"DEBUG_ENDPOINTS"` TLS TLS `yaml:"tls" json:"tls"` } diff --git a/config/config.go b/config/config.go index 54984f8..9f868ce 100644 --- a/config/config.go +++ b/config/config.go @@ -129,13 +129,13 @@ func (cfg *Config) OverrideByRole(role Role) { switch role { case RoleCP: if cfg.Admin.Listen == "" { - cfg.Admin.Listen = "127.0.0.1:8080" + cfg.Admin.Listen = "127.0.0.1:9601" } cfg.Proxy.Listen = "" cfg.Worker.Enabled = false case RoleDPProxy: if cfg.Proxy.Listen == "" { - cfg.Proxy.Listen = "127.0.0.1:8081" + cfg.Proxy.Listen = "0.0.0.0:9600" } cfg.Admin.Listen = "" cfg.Worker.Enabled = false diff --git a/config/proxy.go b/config/proxy.go index 82b9ad5..f4d6d9f 100644 --- a/config/proxy.go +++ b/config/proxy.go @@ -37,7 +37,7 @@ func (cfg Queue) Validate() error { } type ProxyConfig struct { - Listen string `yaml:"listen" json:"listen"` + Listen string `yaml:"listen" json:"listen" default:"0.0.0.0:9600"` TLS TLS `yaml:"tls" json:"tls"` TimeoutRead int64 `yaml:"timeout_read" json:"timeout_read" default:"10" envconfig:"TIMEOUT_READ"` TimeoutWrite int64 `yaml:"timeout_write" json:"timeout_write" default:"10" envconfig:"TIMEOUT_WRITE"` diff --git a/config/status.go b/config/status.go index 90875e2..edf25b4 100644 --- a/config/status.go +++ b/config/status.go @@ -6,7 +6,7 @@ import ( ) type StatusConfig struct { - Listen string `yaml:"listen" json:"listen" default:"127.0.0.1:8082"` + Listen string `yaml:"listen" json:"listen" default:"127.0.0.1:9602"` DebugEndpoints bool `yaml:"debug_endpoints" json:"debug_endpoints" default:"true" envconfig:"DEBUG_ENDPOINTS"` } diff --git a/config/worker.go b/config/worker.go index aec92ba..659aa65 100644 --- a/config/worker.go +++ b/config/worker.go @@ -47,7 +47,7 @@ type Pool struct { } type WorkerConfig struct { - Enabled bool `yaml:"enabled" json:"enabled" default:"false"` + Enabled bool `yaml:"enabled" json:"enabled" default:"true"` Deliverer WorkerDeliverer `yaml:"deliverer" json:"deliverer"` Pool Pool `yaml:"pool" json:"pool"` } diff --git a/docker-compose-clustering.yml b/docker-compose-clustering.yml index cf55e90..ab16440 100644 --- a/docker-compose-clustering.yml +++ b/docker-compose-clustering.yml @@ -23,7 +23,7 @@ services: WEBHOOKX_DATABASE_USERNAME: webhookx WEBHOOKX_DATABASE_DATABASE: webhookx WEBHOOKX_DATABASE_PORT: 5432 - command: webhookx migrations up + command: webhookx db up depends_on: postgres: condition: service_healthy @@ -35,10 +35,10 @@ services: WEBHOOKX_DATABASE_USERNAME: webhookx WEBHOOKX_DATABASE_DATABASE: webhookx WEBHOOKX_DATABASE_PORT: 5432 - WEBHOOKX_ADMIN_LISTEN: 0.0.0.0:8080 + WEBHOOKX_ADMIN_LISTEN: 0.0.0.0:9601 WEBHOOKX_ROLE: cp ports: - - "8080:8080" + - "9601:9601" depends_on: postgres: condition: service_healthy @@ -54,12 +54,12 @@ services: WEBHOOKX_DATABASE_PORT: 5432 WEBHOOKX_REDIS_HOST: redis WEBHOOKX_REDIS_PORT: 6379 - WEBHOOKX_PROXY_LISTEN: 0.0.0.0:8081 + WEBHOOKX_PROXY_LISTEN: 0.0.0.0:9600 WEBHOOKX_PROXY_QUEUE_REDIS_HOST: redis WEBHOOKX_PROXY_QUEUE_REDIS_PORT: 6379 WEBHOOKX_ROLE: dp_proxy ports: - - "8081:8081" + - "9600:9600" depends_on: postgres: condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index cec7cfe..468c8d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: WEBHOOKX_DATABASE_USERNAME: webhookx WEBHOOKX_DATABASE_DATABASE: webhookx WEBHOOKX_DATABASE_PORT: 5432 - command: webhookx migrations up + command: webhookx db up depends_on: webhookx-database: condition: service_healthy @@ -22,16 +22,16 @@ services: WEBHOOKX_DATABASE_PORT: 5432 WEBHOOKX_REDIS_HOST: redis WEBHOOKX_REDIS_PORT: 6379 - WEBHOOKX_ADMIN_LISTEN: 0.0.0.0:8080 + WEBHOOKX_ADMIN_LISTEN: 0.0.0.0:9601 WEBHOOKX_WORKER_ENABLED: true - WEBHOOKX_PROXY_LISTEN: 0.0.0.0:8081 + WEBHOOKX_PROXY_LISTEN: 0.0.0.0:9600 WEBHOOKX_PROXY_QUEUE_REDIS_HOST: redis WEBHOOKX_PROXY_QUEUE_REDIS_PORT: 6379 - WEBHOOKX_STATUS_LISTEN: 0.0.0.0:8082 + WEBHOOKX_STATUS_LISTEN: 0.0.0.0:9602 ports: - - "8080:8080" - - "8081:8081" - - "8082:8082" + - "9601:9601" + - "9600:9600" + - "9602:9602" depends_on: webhookx-database: condition: service_healthy diff --git a/openapi-status.yml b/openapi-status.yml index 93df3c3..7857be5 100644 --- a/openapi-status.yml +++ b/openapi-status.yml @@ -11,7 +11,7 @@ info: version: 0.9.0 servers: - - url: http://localhost:8082 + - url: http://localhost:9602 paths: /: diff --git a/openapi.yml b/openapi.yml index e449a24..42693ad 100644 --- a/openapi.yml +++ b/openapi.yml @@ -11,7 +11,7 @@ info: version: 0.9.0 servers: - - url: http://localhost:8080 + - url: http://localhost:9601 paths: /: diff --git a/test/accesslog/accesslog_test.go b/test/accesslog/accesslog_test.go index 61cb633..1e39f3a 100644 --- a/test/accesslog/accesslog_test.go +++ b/test/accesslog/accesslog_test.go @@ -81,8 +81,6 @@ var _ = Describe("access_log", Ordered, func() { helper.InitDB(true, &entitiesConfig) app = utils.Must(helper.Start(map[string]string{ "NO_COLOR": "true", - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", "WEBHOOKX_ACCESS_LOG_FILE": "webhookx-access.log", })) adminClient = helper.AdminClient() @@ -167,8 +165,6 @@ var _ = Describe("access_log", Ordered, func() { helper.InitDB(true, &entitiesConfig) app = utils.Must(helper.Start(map[string]string{ "NO_COLOR": "true", - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", "WEBHOOKX_ACCESS_LOG_FORMAT": "json", "WEBHOOKX_ACCESS_LOG_FILE": "webhookx-access.log", })) diff --git a/test/admin/attempts_test.go b/test/admin/attempts_test.go index 693010e..0574887 100644 --- a/test/admin/attempts_test.go +++ b/test/admin/attempts_test.go @@ -26,9 +26,7 @@ var _ = Describe("/attempts", Ordered, func() { BeforeAll(func() { db = helper.InitDB(true, nil) - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - })) + app = utils.Must(helper.Start(map[string]string{})) ws = utils.Must(db.Workspaces.GetDefault(context.TODO())) adminClient = helper.AdminClient() }) diff --git a/test/admin/debug_test.go b/test/admin/debug_test.go index 1d3aa37..ae8aa6d 100644 --- a/test/admin/debug_test.go +++ b/test/admin/debug_test.go @@ -17,7 +17,6 @@ var _ = Describe("/debug", Ordered, func() { BeforeAll(func() { app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", "WEBHOOKX_ADMIN_DEBUG_ENDPOINTS": "true", })) adminClient = helper.AdminClient() diff --git a/test/admin/endpoints_test.go b/test/admin/endpoints_test.go index 09ac07e..69b1b40 100644 --- a/test/admin/endpoints_test.go +++ b/test/admin/endpoints_test.go @@ -26,9 +26,7 @@ var _ = Describe("/endpoints", Ordered, func() { db = helper.InitDB(true, nil) var err error adminClient = helper.AdminClient() - app, err = helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - }) + app, err = helper.Start(map[string]string{}) assert.Nil(GinkgoT(), err) ws, err = db.Workspaces.GetDefault(context.TODO()) assert.Nil(GinkgoT(), err) diff --git a/test/admin/events_test.go b/test/admin/events_test.go index ad40595..bd1e9f7 100644 --- a/test/admin/events_test.go +++ b/test/admin/events_test.go @@ -26,9 +26,7 @@ var _ = Describe("/events", Ordered, func() { BeforeAll(func() { db = helper.InitDB(true, nil) - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - })) + app = utils.Must(helper.Start(map[string]string{})) ws = utils.Must(db.Workspaces.GetDefault(context.TODO())) adminClient = helper.AdminClient() }) diff --git a/test/admin/listen_test.go b/test/admin/listen_test.go index bb921c7..19c8c06 100644 --- a/test/admin/listen_test.go +++ b/test/admin/listen_test.go @@ -18,9 +18,7 @@ var _ = Describe("admin", Ordered, func() { BeforeAll(func() { helper.InitDB(true, nil) - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - })) + app = utils.Must(helper.Start(map[string]string{})) adminClient = helper.AdminClient() }) @@ -49,7 +47,6 @@ var _ = Describe("admin", Ordered, func() { BeforeAll(func() { helper.InitDB(true, nil) app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", "WEBHOOKX_ADMIN_TLS_CERT": test.FilePath("server.crt"), "WEBHOOKX_ADMIN_TLS_KEY": test.FilePath("server.key"), })) diff --git a/test/admin/metadata_test.go b/test/admin/metadata_test.go index d434f11..9fc2446 100644 --- a/test/admin/metadata_test.go +++ b/test/admin/metadata_test.go @@ -23,9 +23,7 @@ var _ = Describe("metadata", Ordered, func() { db = helper.InitDB(true, nil) var err error adminClient = helper.AdminClient() - app, err = helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - }) + app, err = helper.Start(map[string]string{}) assert.Nil(GinkgoT(), err) ws, err = helper.GetDeafultWorkspace() assert.Nil(GinkgoT(), err) diff --git a/test/admin/middlewares_test.go b/test/admin/middlewares_test.go index 17af8aa..be069ef 100644 --- a/test/admin/middlewares_test.go +++ b/test/admin/middlewares_test.go @@ -41,9 +41,7 @@ var _ = Describe("middlewares", Ordered, func() { BeforeAll(func() { helper.InitDB(true, nil) adminClient = helper.AdminClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { @@ -76,9 +74,7 @@ var _ = Describe("middlewares", Ordered, func() { adminClient = helper.AdminClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { diff --git a/test/admin/plugins_test.go b/test/admin/plugins_test.go index 316b284..9426e02 100644 --- a/test/admin/plugins_test.go +++ b/test/admin/plugins_test.go @@ -33,9 +33,7 @@ var _ = Describe("/plugins", Ordered, func() { BeforeAll(func() { db = helper.InitDB(true, nil) - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - })) + app = utils.Must(helper.Start(map[string]string{})) ws = utils.Must(db.Workspaces.GetDefault(context.TODO())) adminClient = helper.AdminClient() }) diff --git a/test/admin/sources_test.go b/test/admin/sources_test.go index d36d17d..24c1fa0 100644 --- a/test/admin/sources_test.go +++ b/test/admin/sources_test.go @@ -25,9 +25,7 @@ var _ = Describe("/sources", Ordered, func() { db = helper.InitDB(true, nil) var err error adminClient = helper.AdminClient() - app, err = helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - }) + app, err = helper.Start(map[string]string{}) assert.Nil(GinkgoT(), err) ws, err = db.Workspaces.GetDefault(context.TODO()) assert.Nil(GinkgoT(), err) diff --git a/test/admin/workspaces_test.go b/test/admin/workspaces_test.go index 053eb17..0d80d9a 100644 --- a/test/admin/workspaces_test.go +++ b/test/admin/workspaces_test.go @@ -24,9 +24,7 @@ var _ = Describe("/workspaces", Ordered, func() { db = helper.InitDB(true, nil) var err error adminClient = helper.AdminClient() - app, err = helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - }) + app, err = helper.Start(map[string]string{}) assert.Nil(GinkgoT(), err) ws, err = db.Workspaces.GetDefault(context.TODO()) assert.Nil(GinkgoT(), err) diff --git a/test/cmd/admin_test.go b/test/cmd/admin_test.go index f3c0670..e30e4a6 100644 --- a/test/cmd/admin_test.go +++ b/test/cmd/admin_test.go @@ -47,11 +47,7 @@ var _ = Describe("admin", Ordered, func() { BeforeAll(func() { db = helper.InitDB(true, nil) - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { @@ -166,10 +162,10 @@ var _ = Describe("admin", Ordered, func() { It("--timeout", func() { server := startHTTP(func(writer http.ResponseWriter, r *http.Request) { time.Sleep(time.Second * 2) - }, ":8080") + }, ":9601") output, err := executeCommand(cmd.NewRootCmd(), "admin", "sync", "../fixtures/webhookx.yml", "--timeout", "1") assert.NotNil(GinkgoT(), err) - assert.Equal(GinkgoT(), "Error: Post \"http://localhost:8080/workspaces/default/config/sync\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n", output) + assert.Equal(GinkgoT(), "Error: Post \"http://localhost:9601/workspaces/default/config/sync\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)\n", output) assert.Nil(GinkgoT(), server.Shutdown(context.TODO())) }) @@ -177,11 +173,11 @@ var _ = Describe("admin", Ordered, func() { var url string server := startHTTP(func(writer http.ResponseWriter, r *http.Request) { url = fullURL(r) - }, "127.0.0.1:8080") + }, "127.0.0.1:9601") output, err := executeCommand(cmd.NewRootCmd(), "admin", "sync", "../fixtures/webhookx.yml", "--workspace", "foo") assert.Nil(GinkgoT(), err) assert.Equal(GinkgoT(), "sync successfully\n", output) - assert.Equal(GinkgoT(), "http://localhost:8080/workspaces/foo/config/sync", url) + assert.Equal(GinkgoT(), "http://localhost:9601/workspaces/foo/config/sync", url) assert.Nil(GinkgoT(), server.Shutdown(context.TODO())) }) @@ -207,11 +203,7 @@ var _ = Describe("admin", Ordered, func() { BeforeAll(func() { db = helper.InitDB(true, nil) - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { diff --git a/test/declarative/declarative_test.go b/test/declarative/declarative_test.go index 5a24445..1997170 100644 --- a/test/declarative/declarative_test.go +++ b/test/declarative/declarative_test.go @@ -47,9 +47,7 @@ var _ = Describe("Declarative", Ordered, func() { BeforeAll(func() { helper.InitDB(true, nil) - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - })) + app = utils.Must(helper.Start(map[string]string{})) adminClient = helper.AdminClient() }) diff --git a/test/delivery/acl_test.go b/test/delivery/acl_test.go index 8c5fdb0..613fc0a 100644 --- a/test/delivery/acl_test.go +++ b/test/delivery/acl_test.go @@ -66,8 +66,6 @@ var _ = Describe("network acl", Ordered, func() { proxyClient = helper.ProxyClient() app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", "WEBHOOKX_WORKER_DELIVERER_ACL_DENY": "@default,*.example.com,xn--e1aybc.foo.com", })) @@ -79,7 +77,7 @@ var _ = Describe("network acl", Ordered, func() { }) It("request denied", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -115,7 +113,7 @@ var _ = Describe("network acl", Ordered, func() { }) It("request denied by hostname", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -144,7 +142,7 @@ var _ = Describe("network acl", Ordered, func() { }) It("request denied by unicode hostname", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -173,7 +171,7 @@ var _ = Describe("network acl", Ordered, func() { }) It("request denied by ip resolved by dns", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). diff --git a/test/delivery/delivery_test.go b/test/delivery/delivery_test.go index ba6acd3..99cb383 100644 --- a/test/delivery/delivery_test.go +++ b/test/delivery/delivery_test.go @@ -2,10 +2,8 @@ package delivery import ( "context" - "fmt" "github.com/webhookx-io/webhookx/constants" "github.com/webhookx-io/webhookx/test/helper/factory" - "net" "strconv" "testing" "time" @@ -23,19 +21,6 @@ import ( "github.com/webhookx-io/webhookx/utils" ) -func waitForServer(addr string, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - conn, err := net.DialTimeout("tcp", addr, time.Second) - if err == nil { - _ = conn.Close() - return nil - } - time.Sleep(100 * time.Millisecond) - } - return fmt.Errorf("server at %s not ready after %v", addr, timeout) -} - var _ = Describe("delivery", Ordered, func() { Context("sanity", func() { var proxyClient *resty.Client @@ -59,11 +44,7 @@ var _ = Describe("delivery", Ordered, func() { db = helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { @@ -71,7 +52,7 @@ var _ = Describe("delivery", Ordered, func() { }) It("sanity", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) now := time.Now() @@ -150,11 +131,7 @@ var _ = Describe("delivery", Ordered, func() { db = helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { @@ -162,21 +139,21 @@ var _ = Describe("delivery", Ordered, func() { }) It("all tries are exhausted", func() { - assert.Eventually(GinkgoT(), func() bool { - resp, err := proxyClient.R(). - SetBody(`{ - "event_type": "foo.bar", - "data": { - "key": "value" - } - }`). - Post("/") - return err == nil && resp.StatusCode() == 200 - }, time.Second*5, time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) + assert.NoError(GinkgoT(), err) + + resp, err := proxyClient.R(). + SetBody(`{"event_type": "foo.bar","data": {"key": "value"}}`). + Post("/") + assert.NoError(GinkgoT(), err) + assert.Equal(GinkgoT(), 200, resp.StatusCode()) + eventId := resp.Header().Get(constants.HeaderEventId) time.Sleep(time.Second * 10) - attempts, err := db.Attempts.List(context.TODO(), &query.AttemptQuery{}) + q := query.AttemptQuery{} + q.EventId = &eventId + attempts, err := db.Attempts.List(context.TODO(), &q) assert.NoError(GinkgoT(), err) assert.EqualValues(GinkgoT(), 3, len(attempts)) for i, e := range attempts { @@ -217,11 +194,7 @@ var _ = Describe("delivery", Ordered, func() { db = helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { @@ -282,11 +255,7 @@ var _ = Describe("delivery", Ordered, func() { db = helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { @@ -356,10 +325,7 @@ var _ = Describe("delivery", Ordered, func() { db = helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { @@ -367,7 +333,7 @@ var _ = Describe("delivery", Ordered, func() { }) It("rate limiting", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) for i := 1; i <= 4; i++ { @@ -411,9 +377,7 @@ var _ = Describe("delivery", Ordered, func() { db = helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { @@ -421,7 +385,7 @@ var _ = Describe("delivery", Ordered, func() { }) It("should de-duplicate events by unique_id", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) for i := 1; i <= 2; i++ { resp, err := proxyClient.R(). diff --git a/test/delivery/http_proxy_test.go b/test/delivery/http_proxy_test.go index 2361650..eaa687c 100644 --- a/test/delivery/http_proxy_test.go +++ b/test/delivery/http_proxy_test.go @@ -135,8 +135,6 @@ var _ = Describe("Proxy", Ordered, func() { proxyClient = helper.ProxyClient() app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", "WEBHOOKX_WORKER_DELIVERER_PROXY": httpProxyURL, })) @@ -147,7 +145,7 @@ var _ = Describe("Proxy", Ordered, func() { }) It("http delivery request should be proxied", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -194,7 +192,7 @@ var _ = Describe("Proxy", Ordered, func() { }) It("https delivery request should be proxied", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -240,7 +238,7 @@ var _ = Describe("Proxy", Ordered, func() { }) It("should be failed when connect ", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -296,8 +294,6 @@ var _ = Describe("Proxy", Ordered, func() { proxyClient = helper.ProxyClient() app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", "WEBHOOKX_WORKER_DELIVERER_PROXY": httpsProxyURL, "WEBHOOKX_WORKER_DELIVERER_PROXY_TLS_VERIFY": "TRUE", })) @@ -310,7 +306,7 @@ var _ = Describe("Proxy", Ordered, func() { }) It("http delivery request should be proxied", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -356,7 +352,7 @@ var _ = Describe("Proxy", Ordered, func() { }) It("https delivery request should be proxied", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -427,8 +423,6 @@ var _ = Describe("Proxy", Ordered, func() { proxyClient = helper.ProxyClient() app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", "WEBHOOKX_WORKER_DELIVERER_PROXY": mtlsProxyURL, "WEBHOOKX_WORKER_DELIVERER_PROXY_TLS_CERT": test.FilePath("fixtures/mtls/client.crt"), "WEBHOOKX_WORKER_DELIVERER_PROXY_TLS_KEY": test.FilePath("fixtures/mtls/client.key"), @@ -443,7 +437,7 @@ var _ = Describe("Proxy", Ordered, func() { }) It("http delivery request should be proxied", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -489,7 +483,7 @@ var _ = Describe("Proxy", Ordered, func() { }) It("https delivery request should be proxied", func() { - err := waitForServer("0.0.0.0:8081", time.Second) + err := helper.WaitForServer("0.0.0.0:9600", time.Second) assert.NoError(GinkgoT(), err) resp, err := proxyClient.R(). @@ -539,8 +533,6 @@ var _ = Describe("Proxy", Ordered, func() { Context("error", func() { It("returns error when certificate not found", func() { _, err := helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", "WEBHOOKX_WORKER_DELIVERER_PROXY": mtlsProxyURL, "WEBHOOKX_WORKER_DELIVERER_PROXY_TLS_CERT": test.FilePath("fixtures/mtls/notfound.crt"), "WEBHOOKX_WORKER_DELIVERER_PROXY_TLS_KEY": test.FilePath("fixtures/mtls/client.key"), @@ -552,8 +544,6 @@ var _ = Describe("Proxy", Ordered, func() { }) It("returns error when ca cert not found", func() { _, err := helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", "WEBHOOKX_WORKER_DELIVERER_PROXY": mtlsProxyURL, "WEBHOOKX_WORKER_DELIVERER_PROXY_TLS_CLIENT_CERT": test.FilePath("fixtures/mtls/client.crt"), "WEBHOOKX_WORKER_DELIVERER_PROXY_TLS_CLIENT_KEY": test.FilePath("fixtures/mtls/client.key"), diff --git a/test/helper/helper.go b/test/helper/helper.go index 6a9bf43..628ba4c 100644 --- a/test/helper/helper.go +++ b/test/helper/helper.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "crypto/tls" "encoding/hex" + "fmt" "github.com/go-resty/resty/v2" uuid "github.com/satori/go.uuid" "github.com/webhookx-io/webhookx/app" @@ -17,6 +18,7 @@ import ( "github.com/webhookx-io/webhookx/pkg/log" "github.com/webhookx-io/webhookx/test" "maps" + "net" "os" "regexp" "time" @@ -27,6 +29,14 @@ var ( OtelCollectorMetricsFile = test.FilePath("output/otel/metrics.json") ) +const ( + DefaultHttpProxyURL = "http://localhost:9600" + DefaultHttpsProxyURL = "https://localhost:9600" + DefaultHttpAdminURL = "http://localhost:9601" + DefaultHttpsAdminURL = "https://localhost:9601" + DefaultHttpStatusURL = "http://localhost:9602" +) + var defaultEnvs = map[string]string{ "NO_COLOR": "true", "WEBHOOKX_LOG_LEVEL": "debug", @@ -101,33 +111,33 @@ func Start(envs map[string]string) (application *app.Application, err error) { func AdminClient() *resty.Client { c := resty.New() - c.SetBaseURL("http://localhost:8080") + c.SetBaseURL(DefaultHttpAdminURL) return c } func AdminTLSClient() *resty.Client { c := resty.New() c.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) - c.SetBaseURL("https://localhost:8080") + c.SetBaseURL(DefaultHttpsAdminURL) return c } func ProxyClient() *resty.Client { c := resty.New() - c.SetBaseURL("http://localhost:8081") + c.SetBaseURL(DefaultHttpProxyURL) return c } func ProxyTLSClient() *resty.Client { c := resty.New() c.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) - c.SetBaseURL("https://localhost:8081") + c.SetBaseURL(DefaultHttpsProxyURL) return c } func StatusClient() *resty.Client { c := resty.New() - c.SetBaseURL("http://localhost:8082") + c.SetBaseURL(DefaultHttpStatusURL) return c } @@ -366,3 +376,16 @@ func GenerateTraceID() string { } return hex.EncodeToString(traceID) } + +func WaitForServer(addr string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, time.Second) + if err == nil { + _ = conn.Close() + return nil + } + time.Sleep(100 * time.Millisecond) + } + return fmt.Errorf("server at %s not ready after %v", addr, timeout) +} diff --git a/test/metrics/opentelemetry_test.go b/test/metrics/opentelemetry_test.go index 01495f7..11c5462 100644 --- a/test/metrics/opentelemetry_test.go +++ b/test/metrics/opentelemetry_test.go @@ -37,9 +37,6 @@ var _ = Describe("opentelemetry", Ordered, func() { proxyClient = helper.ProxyClient() var err error app, err = helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", "WEBHOOKX_METRICS_EXPORTS": "opentelemetry", "WEBHOOKX_METRICS_PUSH_INTERVAL": "3", "WEBHOOKX_METRICS_OPENTELEMETRY_PROTOCOL": protocol, diff --git a/test/plugins/function_test.go b/test/plugins/function_test.go index 21eb722..81826e7 100644 --- a/test/plugins/function_test.go +++ b/test/plugins/function_test.go @@ -59,11 +59,7 @@ var _ = Describe("function", Ordered, func() { db = helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { diff --git a/test/plugins/wasm_test.go b/test/plugins/wasm_test.go index 83442b9..bcae8c2 100644 --- a/test/plugins/wasm_test.go +++ b/test/plugins/wasm_test.go @@ -44,11 +44,7 @@ var _ = Describe("wasm", Ordered, func() { db = helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { diff --git a/test/plugins/webhookx_signature_test.go b/test/plugins/webhookx_signature_test.go index 52734ba..a8e3f5d 100644 --- a/test/plugins/webhookx_signature_test.go +++ b/test/plugins/webhookx_signature_test.go @@ -52,11 +52,7 @@ var _ = Describe("webhookx-signature", Ordered, func() { db = helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - "WEBHOOKX_WORKER_ENABLED": "true", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { diff --git a/test/proxy/ingest_test.go b/test/proxy/ingest_test.go index 6174e10..7f04ce9 100644 --- a/test/proxy/ingest_test.go +++ b/test/proxy/ingest_test.go @@ -43,7 +43,7 @@ var _ = Describe("ingest", Ordered, func() { proxyClient = helper.ProxyClient() app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", + "WEBHOOKX_WORKER_ENABLED": "false", })) }) @@ -107,7 +107,6 @@ var _ = Describe("ingest", Ordered, func() { helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", "WEBHOOKX_PROXY_QUEUE_TYPE": "off", "WEBHOOKX_LOG_FILE": "webhookx.log", })) diff --git a/test/proxy/listen_test.go b/test/proxy/listen_test.go index f81e653..b4a635f 100644 --- a/test/proxy/listen_test.go +++ b/test/proxy/listen_test.go @@ -19,9 +19,7 @@ var _ = Describe("proxy", Ordered, func() { BeforeAll(func() { helper.InitDB(true, nil) - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - })) + app = utils.Must(helper.Start(map[string]string{})) proxyClient = helper.ProxyClient() }) @@ -45,7 +43,6 @@ var _ = Describe("proxy", Ordered, func() { BeforeAll(func() { helper.InitDB(true, nil) app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", "WEBHOOKX_PROXY_TLS_CERT": test.FilePath("server.crt"), "WEBHOOKX_PROXY_TLS_KEY": test.FilePath("server.key"), })) diff --git a/test/proxy/ratelimit_test.go b/test/proxy/ratelimit_test.go index b226160..2cfdfaa 100644 --- a/test/proxy/ratelimit_test.go +++ b/test/proxy/ratelimit_test.go @@ -35,9 +35,7 @@ var _ = Describe("rate-limit", Ordered, func() { helper.InitDB(true, &entitiesConfig) proxyClient = helper.ProxyClient() - app = utils.Must(helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", - })) + app = utils.Must(helper.Start(map[string]string{})) }) AfterAll(func() { diff --git a/test/tracing/admin_test.go b/test/tracing/admin_test.go index 87f89a9..adae994 100644 --- a/test/tracing/admin_test.go +++ b/test/tracing/admin_test.go @@ -39,8 +39,6 @@ var _ = Describe("tracing admin", Ordered, func() { adminClient = helper.AdminClient() envs := map[string]string{ - "WEBHOOKX_ADMIN_LISTEN": "0.0.0.0:8080", - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", "WEBHOOKX_TRACING_ENABLED": "true", "WEBHOOKX_TRACING_SAMPLING_RATE": "1.0", "WEBHOOKX_TRACING_OPENTELEMETRY_PROTOCOL": protocol, @@ -70,7 +68,7 @@ var _ = Describe("tracing admin", Ordered, func() { "http.response.body.size": "*", "user_agent.original": "*", "server.address": "localhost", - "server.port": "8080", + "server.port": "9601", "network.protocol.version": "*", "network.peer.address": "*", "network.peer.port": "*", @@ -83,27 +81,22 @@ var _ = Describe("tracing admin", Ordered, func() { "dao.attempts.list": {}, } - // wait for export - proxyFunc := func() bool { - resp, err := proxyClient.R(). - SetBody(`{ - "event_type": "foo.bar", - "data": { - "key": "value" - } - }`).Post("/") - return err == nil && resp.StatusCode() == 200 - } - assert.Eventually(GinkgoT(), proxyFunc, time.Second*5, time.Second) - // make more tracing data - for i := 0; i < 20; i++ { - go proxyFunc() - } + err := helper.WaitForServer("0.0.0.0:9600", time.Second) + assert.NoError(GinkgoT(), err) n, err := helper.FileCountLine(helper.OtelCollectorTracesFile) assert.Nil(GinkgoT(), err) n++ + // make more tracing data + for i := 0; i < 20; i++ { + resp, err := proxyClient.R(). + SetBody(`{"event_type": "foo.bar","data": {"key": "value"}}`). + Post("/") + assert.NoError(GinkgoT(), err) + assert.Equal(GinkgoT(), 200, resp.StatusCode()) + } + assert.Eventually(GinkgoT(), func() bool { resp, err := adminClient.R(). SetHeader("traceparent", fmt.Sprintf("00-%s-0000000000000001-01", traceID)). diff --git a/test/tracing/ginkgo_test.go b/test/tracing/ginkgo_test.go index d9ea94b..1f77739 100644 --- a/test/tracing/ginkgo_test.go +++ b/test/tracing/ginkgo_test.go @@ -30,7 +30,6 @@ var _ = Describe("tracing disabled", Ordered, func() { helper.InitDB(true, &entitiesConfig) envs := map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", "WEBHOOKX_TRACING_ENABLED": "false", "WEBHOOKX_TRACING_SAMPLING_RATE": "1.0", } diff --git a/test/tracing/proxy_test.go b/test/tracing/proxy_test.go index 78d8b9f..e9ac8b7 100644 --- a/test/tracing/proxy_test.go +++ b/test/tracing/proxy_test.go @@ -37,7 +37,6 @@ var _ = Describe("tracing proxy", Ordered, func() { proxyClient = helper.ProxyClient() envs := map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", "WEBHOOKX_TRACING_ENABLED": "true", "WEBHOOKX_TRACING_SAMPLING_RATE": "1.0", "WEBHOOKX_TRACING_ATTRIBUTES": `{"env":"test"}`, @@ -73,7 +72,7 @@ var _ = Describe("tracing proxy", Ordered, func() { "http.response.body.size": "*", "user_agent.original": "*", "server.address": "localhost", - "server.port": "8081", + "server.port": "9600", "network.protocol.version": "*", "network.peer.address": "*", "network.peer.port": "*", @@ -195,7 +194,6 @@ var _ = Describe("tracing proxy", Ordered, func() { proxyClient = helper.ProxyClient() app, err = helper.Start(map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", "WEBHOOKX_TRACING_ENABLED": "true", "WEBHOOKX_TRACING_SAMPLING_RATE": "1", "WEBHOOKX_TRACING_ATTRIBUTES": `{"env":"test"}`, diff --git a/test/tracing/worker_test.go b/test/tracing/worker_test.go index 0b39472..4e7ef11 100644 --- a/test/tracing/worker_test.go +++ b/test/tracing/worker_test.go @@ -33,9 +33,7 @@ var _ = Describe("tracing worker", Ordered, func() { helper.InitDB(true, cfg) proxyClient = helper.ProxyClient() envs := map[string]string{ - "WEBHOOKX_PROXY_LISTEN": "0.0.0.0:8081", "WEBHOOKX_TRACING_ENABLED": "true", - "WEBHOOKX_WORKER_ENABLED": "true", "WEBHOOKX_TRACING_SAMPLING_RATE": "1.0", "WEBHOOKX_TRACING_OPENTELEMETRY_PROTOCOL": protocol, "WEBHOOKX_TRACING_OPENTELEMETRY_ENDPOINT": address, From 86348c636de222763a2d208efd59b0f2468e3294 Mon Sep 17 00:00:00 2001 From: Douglas-Lee Date: Tue, 4 Nov 2025 23:12:34 +0800 Subject: [PATCH 2/3] wip --- config/config.go | 25 ++++++++++++++--- go.mod | 19 +++++++++++-- go.sum | 38 +++++++++++++++++++++++--- pkg/envconfig/env_os.go | 8 ------ pkg/envconfig/env_syscall.go | 8 ------ pkg/secret/reference.go | 52 ++++++++++++++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 25 deletions(-) delete mode 100644 pkg/envconfig/env_os.go delete mode 100644 pkg/envconfig/env_syscall.go create mode 100644 pkg/secret/reference.go diff --git a/config/config.go b/config/config.go index 9f868ce..73067af 100644 --- a/config/config.go +++ b/config/config.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/creasty/defaults" "github.com/webhookx-io/webhookx/pkg/envconfig" + "github.com/webhookx-io/webhookx/pkg/secret" + "github.com/webhookx-io/webhookx/utils" "gopkg.in/yaml.v3" "io" "os" @@ -38,6 +40,7 @@ type Config struct { Tracing TracingConfig `yaml:"tracing" json:"tracing" envconfig:"TRACING"` Role Role `yaml:"role" json:"role" envconfig:"ROLE" default:"standalone"` AnonymousReports bool `yaml:"anonymous_reports" json:"anonymous_reports" envconfig:"ANONYMOUS_REPORTS" default:"true"` + Secret SecretConfig `yaml:"secret" json:"secret" envconfig:"SECRET"` } func (cfg Config) String() string { @@ -82,6 +85,9 @@ func (cfg Config) Validate() error { if !slices.Contains([]Role{RoleStandalone, RoleCP, RoleDPWorker, RoleDPProxy}, cfg.Role) { return fmt.Errorf("invalid role: '%s'", cfg.Role) } + if err := cfg.Secret.Validate(); err != nil { + return err + } return nil } @@ -117,12 +123,25 @@ func InitWithFile(filename string) (*Config, error) { return nil, err } - err = envconfig.Process("WEBHOOKX", &cfg) - if err != nil { + if err := envconfig.Process("WEBHOOKX_SECRET", &cfg.Secret); err != nil { return nil, err } + if cfg.Secret.Provider != "" { + var providerConfig map[string]interface{} + switch cfg.Secret.Provider { + case ProviderAWS: + providerConfig = utils.Must(utils.StructToMap(cfg.Secret.Aws)) + } + manager := secret.NewManager(secret.ProviderType(cfg.Secret.Provider), providerConfig) + err = envconfig.ProcessWithReader("WEBHOOKX", &cfg, + envconfig.ReaderFunc(func(key string) (string, bool, error) { + return manager.Get(key) + })) + return &cfg, err + } - return &cfg, nil + err = envconfig.Process("WEBHOOKX", &cfg) + return &cfg, err } func (cfg *Config) OverrideByRole(role Role) { diff --git a/go.mod b/go.mod index cc1c7ab..3df7bbe 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,9 @@ go 1.25.3 require ( github.com/Masterminds/squirrel v1.5.4 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef + github.com/aws/aws-sdk-go-v2 v1.39.5 + github.com/aws/aws-sdk-go-v2/config v1.31.16 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.39.10 github.com/creasty/defaults v1.8.0 github.com/dop251/goja v0.0.0-20250309171923-bcd7cc6bf64c github.com/elazarl/goproxy v1.7.2 @@ -51,6 +54,17 @@ require github.com/felixge/httpsnoop v1.0.4 // indirect require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect + github.com/aws/smithy-go v1.23.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -62,7 +76,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect @@ -74,6 +88,7 @@ require ( go.opentelemetry.io/contrib/propagators/ot v1.38.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/time v0.12.0 // indirect ) require ( @@ -97,7 +112,7 @@ require ( github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/spf13/pflag v1.0.10 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 go.opentelemetry.io/otel/trace v1.38.0 go.opentelemetry.io/proto/otlp v1.7.1 // indirect diff --git a/go.sum b/go.sum index 2f1e163..0d54b16 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,34 @@ github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrd github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= +github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w= +github.com/aws/aws-sdk-go-v2 v1.39.5/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM= +github.com/aws/aws-sdk-go-v2/config v1.31.16 h1:E4Tz+tJiPc7kGnXwIfCyUj6xHJNpENlY11oKpRTgsjc= +github.com/aws/aws-sdk-go-v2/config v1.31.16/go.mod h1:2S9hBElpCyGMifv14WxQ7EfPumgoeCPZUpuPX8VtW34= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.39.10 h1:S5Mw93I9uFjXnHvkZ19O3Zj0UM5k4v3pYrDZxXCbqUg= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.39.10/go.mod h1:wW/JqWY6yVr88XZJq5wX22l8XNkDdhw+8eDgkN51Rlc= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs= +github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M= +github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -152,8 +180,9 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -201,8 +230,9 @@ github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -276,8 +306,8 @@ golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= diff --git a/pkg/envconfig/env_os.go b/pkg/envconfig/env_os.go deleted file mode 100644 index 6b85e3f..0000000 --- a/pkg/envconfig/env_os.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build appengine || go1.5 -// +build appengine go1.5 - -package envconfig - -import "os" - -var lookupEnv = os.LookupEnv diff --git a/pkg/envconfig/env_syscall.go b/pkg/envconfig/env_syscall.go deleted file mode 100644 index 221ff7f..0000000 --- a/pkg/envconfig/env_syscall.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build !appengine && !go1.5 -// +build !appengine,!go1.5 - -package envconfig - -import "syscall" - -var lookupEnv = syscall.Getenv diff --git a/pkg/secret/reference.go b/pkg/secret/reference.go new file mode 100644 index 0000000..a7a8503 --- /dev/null +++ b/pkg/secret/reference.go @@ -0,0 +1,52 @@ +package secret + +import ( + "errors" + "fmt" + "net/url" + "strings" +) + +var ( + ErrReferenceInvalid = errors.New("invalid reference") +) + +// Reference represents the struct of {secret:///?} +type Reference struct { + Type string + Name string + Properties map[string]string +} + +func Parse(s string) (*Reference, error) { + s = strings.Trim(s, "{}") + s = strings.TrimSpace(s) + u, err := url.Parse(s) + if err != nil { + return nil, err + } + if u.Scheme == "" { + return nil, fmt.Errorf("%w: %q", ErrReferenceInvalid, "missing schema") + } + if u.Host == "" { + return nil, fmt.Errorf("%w: %q", ErrReferenceInvalid, "missing host") + } + if u.Path == "" { + return nil, fmt.Errorf("%w: %q", ErrReferenceInvalid, "missing path") + } + + values, err := url.ParseQuery(u.RawQuery) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrReferenceInvalid, err) + } + + ref := &Reference{ + Type: u.Host, + Name: u.Path, + Properties: make(map[string]string), + } + for k := range values { + ref.Properties[k] = values.Get(k) + } + return ref, err +} From ec00b6891f8553d88d217725c2f3975e374d1cc6 Mon Sep 17 00:00:00 2001 From: Douglas-Lee Date: Wed, 5 Nov 2025 01:17:24 +0800 Subject: [PATCH 3/3] wip --- config.yml | 10 +++ config/config.go | 19 ++++- config/secret.go | 21 ++++++ pkg/envconfig/envconfig.go | 38 +++++++--- pkg/secret/aws.go | 58 +++++++++++++++ pkg/secret/manager.go | 53 ++++++++++++++ pkg/secret/reference.go | 2 +- test/anonymous/anonymous_test.go | 11 +-- test/docker-compose.yml | 11 +++ .../secret-reference/secret-reference_test.go | 71 +++++++++++++++++++ 10 files changed, 275 insertions(+), 19 deletions(-) create mode 100644 config/secret.go create mode 100644 pkg/secret/aws.go create mode 100644 pkg/secret/manager.go create mode 100644 test/secret-reference/secret-reference_test.go diff --git a/config.yml b/config.yml index 3038398..28d63ee 100644 --- a/config.yml +++ b/config.yml @@ -153,3 +153,13 @@ tracing: opentelemetry: protocol: http/protobuf # Supported value: http/protobuf, grpc. endpoint: http://localhost:4318/v1/traces # http/protobuf(http://localhost:4318/v1/traces), grpc(localhost:4317) + + +#------------------------------------------------------------------------------ +# Secret +#------------------------------------------------------------------------------ +#secret: +# provider: aws +# aws: +# region: us-west-1 +# url: http://localhost:4566 diff --git a/config/config.go b/config/config.go index 73067af..ec93969 100644 --- a/config/config.go +++ b/config/config.go @@ -98,12 +98,25 @@ func Init() (*Config, error) { return nil, err } - err := envconfig.Process("WEBHOOKX", &cfg) - if err != nil { + if err := envconfig.Process("WEBHOOKX_SECRET", &cfg.Secret); err != nil { return nil, err } + if cfg.Secret.Provider != "" { + var providerConfig map[string]interface{} + switch cfg.Secret.Provider { + case ProviderAWS: + providerConfig = utils.Must(utils.StructToMap(cfg.Secret.Aws)) + } + manager := secret.NewManager(secret.ProviderType(cfg.Secret.Provider), providerConfig) + err := envconfig.ProcessWithReader("WEBHOOKX", &cfg, + envconfig.ReaderFunc(func(key string) (string, bool, error) { + return manager.Get(key) + })) + return &cfg, err + } - return &cfg, nil + err := envconfig.Process("WEBHOOKX", &cfg) + return &cfg, err } func InitWithFile(filename string) (*Config, error) { diff --git a/config/secret.go b/config/secret.go new file mode 100644 index 0000000..637e5af --- /dev/null +++ b/config/secret.go @@ -0,0 +1,21 @@ +package config + +type Provider string + +const ( + ProviderAWS Provider = "aws" +) + +type SecretConfig struct { + Provider Provider `json:"provider" yaml:"provider"` + Aws AwsProviderConfig `json:"aws" yaml:"aws"` +} + +func (cfg *SecretConfig) Validate() error { + return nil +} + +type AwsProviderConfig struct { + Region string `json:"region" yaml:"region"` + URL string `json:"url" yaml:"url"` +} diff --git a/pkg/envconfig/envconfig.go b/pkg/envconfig/envconfig.go index 5af9fa2..ab90123 100644 --- a/pkg/envconfig/envconfig.go +++ b/pkg/envconfig/envconfig.go @@ -180,30 +180,46 @@ func CheckDisallowed(prefix string, spec interface{}) error { return nil } +type Reader interface { + Read(string) (string, bool, error) +} + +type ReaderFunc func(string) (string, bool, error) + +func (f ReaderFunc) Read(key string) (string, bool, error) { + return f(key) +} + +var ( + EnvironmentReader = ReaderFunc(func(key string) (string, bool, error) { + v, ok := os.LookupEnv(key) + return v, ok, nil + }) +) + // Process populates the specified struct based on environment variables func Process(prefix string, spec interface{}) error { + return ProcessWithReader(prefix, spec, EnvironmentReader) +} + +func ProcessWithReader(prefix string, spec interface{}, reader Reader) error { infos, err := gatherInfo(prefix, spec) for _, info := range infos { - - // `os.Getenv` cannot differentiate between an explicitly set empty value - // and an unset value. `os.LookupEnv` is preferred to `syscall.Getenv`, - // but it is only available in go1.5 or newer. We're using Go build tags - // here to use os.LookupEnv for >=go1.5 - value, ok := lookupEnv(info.Key) - if !ok && info.Alt != "" { - value, ok = lookupEnv(info.Alt) + value, ok, err := reader.Read(info.Key) + if err != nil { + return err // todo warp error? } - //patch: does not handle 'default' tag - def := "" + //patch: do not handle 'default' tag + //def := "" //def := info.Tags.Get("default") //if def != "" && !ok { // value = def //} req := info.Tags.Get("required") - if !ok && def == "" { + if !ok { if isTrue(req) { key := info.Key if info.Alt != "" { diff --git a/pkg/secret/aws.go b/pkg/secret/aws.go new file mode 100644 index 0000000..8717229 --- /dev/null +++ b/pkg/secret/aws.go @@ -0,0 +1,58 @@ +package secret + +import ( + "context" + "errors" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" + "github.com/aws/smithy-go/logging" + "log" +) + +type AwsProvider struct { + cfg interface{} + client *secretsmanager.Client +} + +func NewAwsProvider(cfg map[string]interface{}) *AwsProvider { + opts := []func(*config.LoadOptions) error{ + config.WithClientLogMode(aws.LogRequestWithBody | aws.LogResponseWithBody), + config.WithLogger(logging.LoggerFunc(func(classification logging.Classification, format string, v ...interface{}) { + log.Printf("[AWS %s] %s", classification, fmt.Sprintf(format, v...)) + })), + } + url := cfg["url"].(string) + if url != "" { + opts = append(opts, config.WithBaseEndpoint(url)) + } + region := cfg["region"].(string) + if region != "" { + opts = append(opts, config.WithRegion(region)) + } + + config, err := config.LoadDefaultConfig(context.TODO(), opts...) + if err != nil { + panic(err) // todo + } + + provider := &AwsProvider{} + provider.cfg = cfg + provider.client = secretsmanager.NewFromConfig(config, func(options *secretsmanager.Options) {}) + + return provider +} + +func (p *AwsProvider) GetValue(ctx context.Context, key string, properties map[string]string) (string, error) { + result, err := p.client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{SecretId: aws.String(key)}) + if err != nil { + var awsErr *types.ResourceNotFoundException + if errors.As(err, &awsErr) { + return "", fmt.Errorf("%w: %s", ErrSecretNotFound, key) + } + return "", err + } + return *result.SecretString, nil +} diff --git a/pkg/secret/manager.go b/pkg/secret/manager.go new file mode 100644 index 0000000..8ca2901 --- /dev/null +++ b/pkg/secret/manager.go @@ -0,0 +1,53 @@ +package secret + +import ( + "context" + "errors" + "os" + "strings" +) + +var ( + ErrSecretNotFound = errors.New("secret not found") +) + +type ProviderType string + +const ( + AwsPrivider ProviderType = "aws" +) + +type Provider interface { + GetValue(ctx context.Context, key string, properties map[string]string) (string, error) +} + +type Manager struct { + provider Provider +} + +func NewManager(typ ProviderType, cfg map[string]interface{}) *Manager { + var p Provider + switch typ { + case AwsPrivider: + p = NewAwsProvider(cfg) + } + return &Manager{ + provider: p, + } +} + +func (p *Manager) Get(key string) (string, bool, error) { + value, ok := os.LookupEnv(key) + if ok && strings.HasPrefix(value, "{secret://") && strings.HasSuffix(value, "}") { + ref, err := Parse(value) + if err != nil { + return "", false, err + } + value, err = p.provider.GetValue(context.TODO(), ref.Name, ref.Properties) + if err != nil { + return "", false, err + } + return value, true, nil + } + return value, ok, nil +} diff --git a/pkg/secret/reference.go b/pkg/secret/reference.go index a7a8503..c79d6f9 100644 --- a/pkg/secret/reference.go +++ b/pkg/secret/reference.go @@ -42,7 +42,7 @@ func Parse(s string) (*Reference, error) { ref := &Reference{ Type: u.Host, - Name: u.Path, + Name: u.Path[1:], Properties: make(map[string]string), } for k := range values { diff --git a/test/anonymous/anonymous_test.go b/test/anonymous/anonymous_test.go index f3b8c88..dbe57b1 100644 --- a/test/anonymous/anonymous_test.go +++ b/test/anonymous/anonymous_test.go @@ -8,6 +8,7 @@ import ( "github.com/webhookx-io/webhookx/test/helper" "github.com/webhookx-io/webhookx/utils" "testing" + "time" ) var _ = Describe("anonymous reports", Ordered, func() { @@ -18,7 +19,7 @@ var _ = Describe("anonymous reports", Ordered, func() { BeforeAll(func() { helper.InitDB(true, nil) app = utils.Must(helper.Start(map[string]string{ - "ANONYMOUS_REPORTS": "false", + "WEBHOOKX_ANONYMOUS_REPORTS": "false", })) }) @@ -27,9 +28,11 @@ var _ = Describe("anonymous reports", Ordered, func() { }) It("should display log when anonymous_reports is disabled", func() { - matched, err := helper.FileHasLine("webhookx.log", "^.*anonymous reports is disabled$") - assert.Nil(GinkgoT(), err) - assert.Equal(GinkgoT(), true, matched) + assert.Eventually(GinkgoT(), func() bool { + matched, err := helper.FileHasLine("webhookx.log", "^.*anonymous reports is disabled$") + assert.Nil(GinkgoT(), err) + return matched + }, time.Second, time.Millisecond*100) }) }) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 1578152..e8d28c1 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -43,3 +43,14 @@ services: ports: - 4317:4317 - 4318:4318 + + localstack: + container_name: "localstack" + image: localstack/localstack + ports: + - "4566:4566" + environment: + DEBUG: "1" +# DOCKER_HOST: unix:///var/run/docker.sock +# volumes: +# - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/test/secret-reference/secret-reference_test.go b/test/secret-reference/secret-reference_test.go new file mode 100644 index 0000000..3e888c0 --- /dev/null +++ b/test/secret-reference/secret-reference_test.go @@ -0,0 +1,71 @@ +package secret_reference + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + "github.com/webhookx-io/webhookx/app" + "github.com/webhookx-io/webhookx/test/helper" + "testing" +) + +var _ = Describe("SecretReference", Ordered, func() { + + Context("test", func() { + var app *app.Application + + AfterEach(func() { + app.Stop() + }) + + BeforeAll(func() { + helper.InitDB(true, nil) + // mock aws config + config, err := config.LoadDefaultConfig(context.TODO(), + config.WithBaseEndpoint("http://localhost:4566"), + config.WithRegion("us-east-1"), + config.WithCredentialsProvider(aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider( + "test", + "test", + "", + ))), + ) + assert.NoError(GinkgoT(), err) + client := secretsmanager.NewFromConfig(config, func(options *secretsmanager.Options) {}) + _, err = client.CreateSecret(context.TODO(), &secretsmanager.CreateSecretInput{ + Name: aws.String("path/to/value"), + SecretString: aws.String("this is value"), + }) + assert.NoError(GinkgoT(), err) + }) + + AfterAll(func() { + app.Stop() + }) + + It("reference should be de-reference", func() { + var err error + app, err = helper.Start(map[string]string{ + "AWS_ACCESS_KEY_ID": "test", + "AWS_SECRET_ACCESS_KEY": "test", + "WEBHOOKX_SECRET_AWS_REGION": "us-east-1", + "WEBHOOKX_SECRET_AWS_URL": "http://localhost:4566", + "WEBHOOKX_SECRET_PROVIDER": "aws", + "WEBHOOKX_REDIS_PASSWORD": "{secret://aws/path/to/value}", + }) + assert.Nil(GinkgoT(), err) + assert.Equal(GinkgoT(), "this is value", string(app.Config().Redis.Password)) + }) + }) + +}) + +func TestAdmin(t *testing.T) { + gomega.RegisterFailHandler(Fail) + RunSpecs(t, "SecretReference Suite") +}