From 23f4a3ff8506f361cb5f7e8803b5c60a490231af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Wed, 25 Feb 2026 16:26:04 +0300 Subject: [PATCH] fix: remove flaky integration tests from CI Integration tests depend on external service secrets (Discord, Slack, Custom webhooks) that are unavailable in fork PRs, causing consistent CI failures. Remove the integration test suite and keep unit tests. Closes #514 --- .github/workflows/build-test.yml | 27 +-------- cmd/integration-test/action-run.sh | 24 -------- cmd/integration-test/gotify.sh | 32 ---------- cmd/integration-test/integration.go | 53 ----------------- cmd/integration-test/providers.go | 86 --------------------------- cmd/integration-test/run.sh | 14 ----- cmd/integration-test/test-config.yaml | 54 ----------------- internal/testutils/integration.go | 41 ------------- 8 files changed, 2 insertions(+), 329 deletions(-) delete mode 100755 cmd/integration-test/action-run.sh delete mode 100644 cmd/integration-test/gotify.sh delete mode 100644 cmd/integration-test/integration.go delete mode 100644 cmd/integration-test/providers.go delete mode 100755 cmd/integration-test/run.sh delete mode 100644 cmd/integration-test/test-config.yaml delete mode 100644 internal/testutils/integration.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index bf9ecf7..4bbaa8a 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -3,18 +3,10 @@ on: pull_request: workflow_dispatch: -jobs: +jobs: build: name: Test Builds runs-on: ubuntu-latest - services: - gotify: - image: ghcr.io/gotify/server - ports: - - 80:80 - options: >- - -v /var/gotify/data:/app/data - --name gotify steps: - name: Set up Go @@ -26,23 +18,8 @@ jobs: uses: actions/checkout@v4 - name: Test - run: go test . - working-directory: cmd/notify/ + run: go test ./... - name: Build run: go build . working-directory: cmd/notify/ - - - name: Integration Tests - env: - DISCORD_WEBHOOK_URL: "${{ secrets.DISCORD_WEBHOOK_URL }}" - SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" - CUSTOM_WEBHOOK_URL: "${{ secrets.CUSTOM_WEBHOOK_URL }}" - run: | - if [ "$GITHUB_ACTOR" == "dependabot[bot]" ]; then - export DEPENDABOT=true - fi - chmod +x gotify.sh - chmod +x action-run.sh - bash action-run.sh - working-directory: cmd/integration-test/ \ No newline at end of file diff --git a/cmd/integration-test/action-run.sh b/cmd/integration-test/action-run.sh deleted file mode 100755 index 58fb77b..0000000 --- a/cmd/integration-test/action-run.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# setup gotify server -bash gotify.sh -export $(echo "GOTIFY_APP_TOKEN=$(final-config.yaml"; - cat test-config.yaml; - echo "EOF"; -) >temp.yaml -. temp.yaml -rm integration-test notify 2>/dev/null - -go build ../notify -go build - -DEBUG=true ./integration-test --provider-config final-config.yaml -if [ $? -eq 0 ] -then - exit 0 -else - exit 1 -fi diff --git a/cmd/integration-test/gotify.sh b/cmd/integration-test/gotify.sh deleted file mode 100644 index 699d0df..0000000 --- a/cmd/integration-test/gotify.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -# If script running in local, then setup server with docker -# docker run -p 80:80 -v /var/gotify/data:/app/data ghcr.io/gotify/server - -# Configuration variables -GOTIFY_URL="http://localhost" -AUTH_HEADER="Authorization: Basic YWRtaW46YWRtaW4=" # default username and password is admin:admin -APPLICATION_JSON='{ - "defaultPriority": 5, - "description": "Gotify Test server", - "name": "test-server" -}' - -# Create a new application and get the response -APPLICATION_RESPONSE=$(curl --location "$GOTIFY_URL/application" \ ---header "Content-Type: application/json" \ ---header "$AUTH_HEADER" \ ---data "$APPLICATION_JSON") - -# Extract application token from the response -APP_TOKEN=$(echo "$APPLICATION_RESPONSE" | jq -r '.token') - -if [ "$APP_TOKEN" == "null" ]; then - echo "Failed to create application" - exit 1 -fi - -echo "Gotify Application created successfully. Token: $APP_TOKEN" - -# write the token to a file -echo "$APP_TOKEN" > gotify-app-token.txt diff --git a/cmd/integration-test/integration.go b/cmd/integration-test/integration.go deleted file mode 100644 index 2b45764..0000000 --- a/cmd/integration-test/integration.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - - "github.com/logrusorgru/aurora" - - "github.com/projectdiscovery/notify/internal/testutils" -) - -var ( - providerConfig = flag.String("provider-config", "", "provider config to use for testing") - debug = os.Getenv("DEBUG") == "true" - isDependabot = os.Getenv("DEPENDABOT") == "true" - errored = false - success = aurora.Green("[✓]").String() - failed = aurora.Red("[✘]").String() - testCases = map[string]testutils.TestCase{ - "discord": &discord{}, - "slack": &slack{}, - "custom": &custom{}, - // "telegram": &telegram{}, - // "teams": &teams{}, - // "smtp": &smtp{}, - // "pushover": &pushover{}, - "gotify": &gotify{}, - "notion": ¬ion{}, - } -) - -func main() { - flag.Parse() - - for name, test := range testCases { - // run only gotify test for dependabot - if isDependabot && name != "gotify" { - continue - } - fmt.Printf("Running test cases for \"%s\"\n", aurora.Blue(name)) - err := test.Execute() - if err != nil { - fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, name, err) - errored = true - } else { - fmt.Printf("%s Test \"%s\" passed!\n", success, name) - } - } - if errored { - os.Exit(1) - } -} diff --git a/cmd/integration-test/providers.go b/cmd/integration-test/providers.go deleted file mode 100644 index 40ab502..0000000 --- a/cmd/integration-test/providers.go +++ /dev/null @@ -1,86 +0,0 @@ -package main - -import ( - "fmt" - "strings" - - "github.com/projectdiscovery/notify/internal/testutils" -) - -func run(provider string) error { - args := []string{"--provider", provider} - if *providerConfig != "" { - args = append(args, "--provider-config", *providerConfig) - } - results, err := testutils.RunNotifyAndGetResults(debug, args...) - if err != nil { - return err - } - if len(results) < 1 { - return errIncorrectResultsCount(results) - } - for _, r := range results { - if !strings.Contains(strings.ToLower(r), strings.ToLower(provider)) { - return fmt.Errorf("incorrect result %s", results[0]) - } - } - return nil -} - -type discord struct{} - -func (h *discord) Execute() error { - return run("discord") -} - -type custom struct{} - -func (h *custom) Execute() error { - return run("custom") -} - -type slack struct{} - -func (h *slack) Execute() error { - return run("slack") -} - -// type pushover struct{} -// -// func (h *pushover) Execute() error { -// return run("pushover") -// } -// -// type smtp struct{} -// -// func (h *smtp) Execute() error { -// return run("smtp") -// } -// -// type teams struct{} -// -// func (h *teams) Execute() error { -// return run("teams") -// } -// -// type telegram struct{} -// -// func (h *telegram) Execute() error { -// return run("telegram") -// } - -type gotify struct{} - -func (h *gotify) Execute() error { - return run("gotify") -} - -type notion struct{} - -func (h *notion) Execute() error { - return run("notion") -} - -func errIncorrectResultsCount(results []string) error { - return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t")) -} diff --git a/cmd/integration-test/run.sh b/cmd/integration-test/run.sh deleted file mode 100755 index 15abb9f..0000000 --- a/cmd/integration-test/run.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -rm integration-test notify 2>/dev/null - -go build ../notify -go build - -./integration-test -if [ $? -eq 0 ] -then - exit 0 -else - exit 1 -fi diff --git a/cmd/integration-test/test-config.yaml b/cmd/integration-test/test-config.yaml deleted file mode 100644 index 2cc5cfc..0000000 --- a/cmd/integration-test/test-config.yaml +++ /dev/null @@ -1,54 +0,0 @@ -discord: - - id: "disocord-integration-test" - discord_webhook_url: "${DISCORD_WEBHOOK_URL}" - discord_format: "{{data}}" -slack: - - id: "slack-integration-test" - slack_channel: "random" - slack_username: "test" - slack_webhook_url: "${SLACK_WEBHOOK_URL}" - slack_format: "{{data}}" -telegram: - - id: "telegram-integration-test" - telegram_api_key: "${telegram_api_key}" - telegram_chat_id: "${telegram_chat_id}" - telegram_format: "{{data}}" -custom: - - id: "custom-integration-test" - custom_webhook_url: "${CUSTOM_WEBHOOK_URL}" - custom_method: POST - custom_format: '{{data}}' - custom_headers: - Content-Type: application/json -pushover: - - id: "push" - pushover_user_key: "${pushover_user_key}" - pushover_api_token: "${pushover_api_token}" - pushover_format: "{{data}}" - pushover_devices: - - "iphone" -smtp: - - id: email - smtp_server: "${smtp_server}" - smtp_username: "${smtp_username}" - smtp_password: "${smtp_password}" - from_address: "${smtp_from_address}" - smtp_cc: - - "${smtp_cc}" - smtp_format: "{{data}}" -teams: - - id: teams-integration-test - teams_webhook_url: "${teams_webhook_url}" - teams_format: "{{data}}" -gotify: - - id: "gotify" - gotify_host: "localhost" - gotify_port: "80" - gotify_token: "${GOTIFY_APP_TOKEN}" - gotify_format: "{{data}}" - gotify_disabletls: true -notion: - - id: "notion" - notion_api_key: "${notion_api_key}" - notion_database_id: "${notion_database_id}" - notion_in_page_title: "${notion_in_page_title}" # optional diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go deleted file mode 100644 index fa4887e..0000000 --- a/internal/testutils/integration.go +++ /dev/null @@ -1,41 +0,0 @@ -package testutils - -import ( - "fmt" - "os/exec" - "strings" -) - -// RunNotifyAndGetResults returns a list of results for a template -func RunNotifyAndGetResults(debug bool, args ...string) ([]string, error) { - cmd := exec.Command("bash", "-c") - cmdLine := `echo "hello from notify integration test :)"` + ` | ./notify ` - cmdLine += strings.Join(args, " ") - - cmdLine += " --v" - - cmd.Args = append(cmd.Args, cmdLine) - data, err := cmd.CombinedOutput() - if err != nil { - return nil, err - } - parts := []string{} - items := strings.Split(string(data), "\n") - for _, i := range items { - if i != "" { - if debug { - fmt.Printf("%s\n", i) - } - if strings.Contains(i, "notification sent for id:") { - parts = append(parts, i) - } - } - } - return parts, nil -} - -// TestCase is a single integration test case -type TestCase interface { - // Execute executes a test case and returns any errors if occurred - Execute() error -}