diff --git a/tools/e2e-tester/.gitignore b/tools/e2e-tester/.gitignore new file mode 100644 index 00000000..2e3135c8 --- /dev/null +++ b/tools/e2e-tester/.gitignore @@ -0,0 +1,45 @@ +# Copyright 2025 Deutsche Telekom IT GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin/* +Dockerfile.cross +admin_data +*.crt +.env* +access-token +.env +snapshotter-config.yaml + +snapshots/ + +# Test binary, built with `go test -c` +*.test + +# Output of the go/ginkgo coverage tool, specifically when used with LiteIDE +*.out +coverage.html +ginkgo.report + +# Go workspace file +go.work +go.work.* + +# Kubernetes Generated files - skip generated files, except for vendored files +!vendor/**/zz_generated.* + +# editor and IDE paraphernalia +.idea +.vscode +*.swp +*.swo +*~ +.jfrog +test/load/* + diff --git a/tools/e2e-tester/README.md b/tools/e2e-tester/README.md index 59835f82..4ea7c786 100644 --- a/tools/e2e-tester/README.md +++ b/tools/e2e-tester/README.md @@ -79,7 +79,7 @@ roverctl: environments: - name: "test-env" - token: "env:TEST_TOKEN" # Use environment variable TEST_TOKEN + token: "env://TEST_TOKEN" # Use environment variable TEST_TOKEN suites: - name: "basic-suite" @@ -87,7 +87,7 @@ suites: - "test-env" cases: - name: "version-check" - must_pass: true + run_policy: critical command: "--version" compare: true ``` @@ -108,6 +108,8 @@ export TEST_TOKEN="your-token-value" The E2E-Tester is configured through a YAML file that specifies all testing parameters, environments, and test cases. +> **Tip:** JSON schemas are available in `schemas/` for editor autocompletion and validation. Add `# yaml-language-server: $schema=./schemas/config.schema.json` at the top of your config file. + ### Configuration Structure ```yaml @@ -126,9 +128,9 @@ roverctl: # Test environments environments: - name: "team-a" # Unique name for the environment - token: "env:TEAM_A_TOKEN" # Token for authentication, can use env variable with "env:" prefix + token: "env://TEAM_A_TOKEN" # Token for authentication, can use env variable with "env:" prefix - name: "team-b" - token: "env:TEAM_B_TOKEN" + token: "env://TEAM_B_TOKEN" # Test suites suites: @@ -147,7 +149,7 @@ Each environment represents a separate context with its own authentication: ```yaml environments: - name: "production" # Environment identifier - token: "env:PROD_TOKEN" # Reference to environment variable + token: "env://PROD_TOKEN" # Reference to environment variable - name: "staging" token: "direct-token-value" # Direct token value (not recommended for security) ``` @@ -176,6 +178,40 @@ cases: command: "dangerous-operation" ``` +### External Suite Files + +For larger configurations, suites can be defined in separate files to keep the root config slim: + +```yaml +# Root config +suites: + - name: "api-validation" + filepath: "./suites/api-validation.yaml" # Relative to config file + + - name: "inline-suite" + environments: ["team-a"] + cases: + - name: "version-check" + command: "--version" +``` + +The external file contains the suite content (cases, environments, description): + +```yaml +# suites/api-validation.yaml +environments: + - "team-a" +cases: + - name: "apply-config" + command: "apply -f ./examples/test-files" + compare: true + - name: "cleanup" + run_policy: always + command: "delete -f ./examples/test-files" +``` + +> **Note:** `filepath` and `cases` are mutually exclusive. Paths are resolved relative to the root config file. + ### Test Case Configuration Each test case defines a specific command to run and how to validate its output: @@ -185,7 +221,7 @@ cases: - name: "version-check" # Name of the test case description: "Verify rover-ctl version" # Optional description type: "roverctl" # Type of command (default: "roverctl") - must_pass: true # Whether this test must pass for suite to succeed + run_policy: critical # Execution policy: "normal", "critical", or "always" command: "--version" # Command to execute compare: true # Whether to compare with snapshot wait_before: 5s # Optional: Wait before executing @@ -194,6 +230,44 @@ cases: selector: "$.version" # Optional: JSON path selector for partial output comparison ``` +### Run Policy + +The `run_policy` field controls test execution behavior relative to prior test failures: + +| Policy | Runs after prior failure? | On ERROR status | +|--------|---------------------------|-----------------| +| `normal` | ❌ **Skipped** | Suite continues | +| `critical` | ✅ Runs | **Aborts suite** | +| `always` | ✅ Runs | Suite continues | + +**When to use each policy:** +- `normal` (default): Regular tests that should be skipped if something already broke +- `critical`: Important tests that must run AND whose failure should stop everything +- `always`: Cleanup/teardown that must execute no matter what happened before + +**Examples:** + +```yaml +cases: + # Critical test - suite aborts if this fails with an error + - name: "version-check" + run_policy: critical + command: "--version" + compare: true + + # Normal test - skipped if prior tests failed (default when omitted) + - name: "get-info" + # run_policy: normal (default) + command: "get-info --name test-rover" + compare: true + + # Cleanup test - always runs to clean up resources + - name: "delete-resources" + run_policy: always + command: "delete -f ./examples/test-files" + compare: true +``` + Special test case types: ```yaml @@ -272,12 +346,12 @@ Test cases are the building blocks of the E2E-Tester. They define individual com Each test case must have: 1. A unique name within its suite 2. A command to execute -3. Whether the command must pass for the test to succeed +3. A run policy defining execution behavior (optional, defaults to `normal`) 4. Whether to compare output with a snapshot ```yaml - name: "version-check" - must_pass: true + run_policy: critical command: "--version" compare: true ``` @@ -306,7 +380,7 @@ You can enhance test cases with additional parameters: ```yaml - name: "complex-test" description: "Tests complex API functionality" - must_pass: true + run_policy: critical command: "apply -f ./config/complex-api.yaml" compare: true wait_before: 5s # Wait before execution @@ -321,7 +395,7 @@ You can enhance test cases with additional parameters: #### Basic Version Check ```yaml - name: "version-check" - must_pass: true + run_policy: critical command: "--version" compare: true ``` @@ -329,13 +403,13 @@ You can enhance test cases with additional parameters: #### Resource Creation and Verification ```yaml - name: "create-resource" - must_pass: true + run_policy: critical command: "apply -f ./examples/test-files/resource.yaml" compare: true wait_after: 2s - name: "verify-resource" - must_pass: true + run_policy: critical command: "get-info --name test-resource" compare: true ``` @@ -344,17 +418,17 @@ You can enhance test cases with additional parameters: ```yaml - name: "system-state-snapshot" type: "snapshot" - must_pass: false + # run_policy: normal (default - skipped if prior tests failed) command: "snap --source dataplane1 --route api-route-v1" compare: true selector: "$.b" wait_before: 5s ``` -#### Cleanup +#### Cleanup (Always Runs) ```yaml - name: "delete-resources" - must_pass: false # Marking as non-critical for test success + run_policy: always # Always runs to ensure cleanup happens command: "delete -f ./examples/test-files" compare: true ``` @@ -363,7 +437,7 @@ You can enhance test cases with additional parameters: ```yaml - name: "production-only-test" environment: "production" - must_pass: true + run_policy: critical command: "special-command --production-flag" compare: true ``` @@ -406,7 +480,7 @@ To create a snapshot test case, use the `snapshot` type: ```yaml - name: "api-route-snapshot" type: "snapshot" # Indicates this is a snapshotter operation - must_pass: true + run_policy: critical command: "snap --source production-gateway --route api-route-v1" compare: true selector: "$.b" # Snapshotter puts the new snapshot in $.b @@ -464,23 +538,23 @@ suites: - "team-a" cases: - name: "version-check" - must_pass: true + run_policy: critical command: "--version" compare: true - name: "apply-config" - must_pass: true + run_policy: critical command: "apply -f ./examples/test-files" compare: true wait_after: 2s - name: "get-info" - must_pass: true + run_policy: critical command: "get-info --name test-rover" compare: true - name: "delete" - must_pass: true + run_policy: always # Cleanup always runs command: "delete -f ./examples/test-files" compare: true ``` @@ -492,9 +566,9 @@ This workflow tests the same operations across multiple environments: ```yaml environments: - name: "development" - token: "env:DEV_TOKEN" + token: "env://DEV_TOKEN" - name: "staging" - token: "env:STAGING_TOKEN" + token: "env://STAGING_TOKEN" suites: - name: "cross-environment-validation" @@ -503,18 +577,18 @@ suites: - "staging" cases: - name: "apply-config" - must_pass: true + run_policy: critical command: "apply -f ./examples/test-files" compare: true wait_after: 2s - name: "get-info" - must_pass: true + run_policy: critical command: "get-info --name test-rover" compare: true - name: "delete" - must_pass: true + run_policy: always # Cleanup always runs command: "delete -f ./examples/test-files" compare: true ``` @@ -530,32 +604,32 @@ suites: - "production" cases: - name: "apply-route" - must_pass: true + run_policy: critical command: "apply -f ./examples/gateway/route.yaml" compare: true wait_after: 5s - name: "snapshot-route-state" type: "snapshot" - must_pass: true + run_policy: critical command: "snap --source prod-gateway --route my-api-route" compare: true selector: "$.b" wait_before: 2s - name: "verify-route-exists" - must_pass: true + run_policy: critical command: "get-info --name my-api-route" compare: true - name: "delete-route" - must_pass: true + run_policy: always # Cleanup always runs command: "delete -f ./examples/gateway/route.yaml" compare: true - name: "verify-route-deleted" type: "snapshot" - must_pass: true + run_policy: always # Verification after cleanup command: "snap --source prod-gateway --route my-api-route" compare: true selector: "$.b" diff --git a/tools/e2e-tester/examples/basic-config.yaml b/tools/e2e-tester/examples/basic-config.yaml index ae6225e6..85ff1af1 100644 --- a/tools/e2e-tester/examples/basic-config.yaml +++ b/tools/e2e-tester/examples/basic-config.yaml @@ -2,6 +2,8 @@ # # SPDX-License-Identifier: Apache-2.0 +# yaml-language-server: $schema=../schemas/config.schema.json + # Example E2E Test Configuration # Snapshotter configuration @@ -18,45 +20,14 @@ roverctl: # Test environments environments: - name: "team-a" - token: "env:TEAM_A_TOKEN" + token: "env://TEAM_A_TOKEN" - name: "team-b" - token: "env:TEAM_B_TOKEN" + token: "env://TEAM_B_TOKEN" # Test suites suites: - name: "basic-suite" - environments: - - "team-a" - cases: - - name: "version-check" - type: "roverctl" # Type is optional, defaults to "roverctl" - must_pass: true - command: "--version" - compare: true - - - name: "apply-config" - must_pass: true - command: "apply -f ./examples/test-files" - compare: true - wait_after: 2s - - - name: "get-info" - must_pass: true - command: "get-info -f ./examples/test-files/rover.yaml" - compare: true - - - name: "system-state-snapshot" - type: "snapshot" - must_pass: false - command: "snap --no-store --format yaml --source poc-dataplane1 --route poc--eni-e2e-test-v1" - compare: true - selector: "$.b" # snapshotter has new snapshot in $.b and old in $.a - wait_before: 30s - - - name: "delete" - must_pass: false - command: "delete -f ./examples/test-files" - compare: true + filepath: "basic-suite.yaml" # External suite file relative to this config-file - name: "multi-env-suite" environments: @@ -64,7 +35,7 @@ suites: - "team-b" cases: - name: "version-check" - must_pass: true + run_policy: critical command: "--version" compare: true @@ -76,22 +47,22 @@ suites: cases: - name: "apply-config" environment: team-a - must_pass: false + # run_policy: normal (default) command: "apply -f ./examples/super-long-test-files" compare: true - name: "get-info" environment: team-a - must_pass: false + # run_policy: normal (default) command: "get-info --name test-rover" compare: true - name: "apply-config" environment: team-b description: | - Applying this will result in a blocked API as the same API path + Applying this will result in a blocked API as the same API path is already exposed by team-a - must_pass: false + # run_policy: normal (default) command: "apply -f ./examples/super-long-test-files" compare: true @@ -99,7 +70,7 @@ suites: environment: team-b description: | This must include an error message indicating the API is blocked - must_pass: false + # run_policy: normal (default) command: "get-info --name test-rover" compare: true @@ -107,7 +78,7 @@ suites: environment: team-a description: | Remove the exposure from team-a so that team-b becomes active - must_pass: false + run_policy: always # Cleanup always runs command: "delete -f ./examples/super-long-test-files" compare: true wait_after: 1s @@ -116,12 +87,12 @@ suites: environment: team-b description: | This must now succeed as team-b is active - must_pass: false + # run_policy: normal (default) command: "get-info --name test-rover" compare: true - name: "delete" environment: team-b - must_pass: false + run_policy: always # Cleanup always runs command: "delete -f ./examples/super-long-test-files" compare: true \ No newline at end of file diff --git a/tools/e2e-tester/examples/basic-suite.yaml b/tools/e2e-tester/examples/basic-suite.yaml new file mode 100644 index 00000000..a114b014 --- /dev/null +++ b/tools/e2e-tester/examples/basic-suite.yaml @@ -0,0 +1,37 @@ +# Copyright 2026 Deutsche Telekom IT GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +# yaml-language-server: $schema=../schemas/suitecontent.schema.json +environments: + - "team-a" +cases: + - name: "version-check" + type: "roverctl" # Type is optional, defaults to "roverctl" + run_policy: critical + command: "--version" + compare: true + + - name: "apply-config" + run_policy: critical + command: "apply -f ./examples/test-files" + compare: true + wait_after: 2s + + - name: "get-info" + run_policy: critical + command: "get-info -f ./examples/test-files/rover.yaml" + compare: true + + - name: "system-state-snapshot" + type: "snapshot" + # run_policy: normal (default - skipped if prior tests failed) + command: "snap --no-store --format yaml --source poc-dataplane1 --route poc--eni-e2e-test-v1" + compare: true + selector: "$.b" # snapshotter has new snapshot in $.b and old in $.a + wait_before: 30s + + - name: "delete" + run_policy: always # Cleanup always runs regardless of prior failures + command: "delete -f ./examples/test-files" + compare: true \ No newline at end of file diff --git a/tools/e2e-tester/go.mod b/tools/e2e-tester/go.mod index fdb37e91..b6b58b0b 100644 --- a/tools/e2e-tester/go.mod +++ b/tools/e2e-tester/go.mod @@ -4,10 +4,11 @@ module github.com/telekom/controlplane/tools/e2e-tester -go 1.24.9 +go 1.25.5 require ( github.com/fatih/color v1.18.0 + github.com/go-playground/validator/v10 v10.30.1 github.com/goccy/go-yaml v1.19.0 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.10.1 @@ -17,13 +18,22 @@ require ( ) require ( + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect github.com/gkampitakis/go-diff v1.3.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/jsonschema v0.13.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/ron96g/json-schema-gen v0.3.1 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect @@ -34,12 +44,15 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace github.com/telekom/controlplane => ../../ - replace github.com/telekom/controlplane/tools/snapshotter => ../snapshotter + +tool github.com/ron96g/json-schema-gen diff --git a/tools/e2e-tester/go.sum b/tools/e2e-tester/go.sum index 5970cc1d..45518698 100644 --- a/tools/e2e-tester/go.sum +++ b/tools/e2e-tester/go.sum @@ -1,3 +1,7 @@ +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -7,8 +11,18 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE= @@ -17,10 +31,17 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= +github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -33,6 +54,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/ron96g/json-schema-gen v0.3.1 h1:93h6j44AtMuYRHv9PTJcKtTFkTeJ69SKHfh9DskTLFI= +github.com/ron96g/json-schema-gen v0.3.1/go.mod h1:nzJ+KkoW5ZX/Hk8NnqWoNdnK8RKOJddCGUP8BGSo4nA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= @@ -63,6 +86,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -71,11 +96,13 @@ go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/e2e-tester/main.go b/tools/e2e-tester/main.go index fbeab30e..e9182864 100644 --- a/tools/e2e-tester/main.go +++ b/tools/e2e-tester/main.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "os/signal" + "path/filepath" "time" "github.com/spf13/cobra" @@ -32,22 +33,82 @@ var ( envFilter string ) -func main() { - rootCmd := &cobra.Command{ +var ( + cfg config.Config + + runCmd = &cobra.Command{ + Use: "run", + Short: "Run the end-to-end tests", + Long: `Executes the end-to-end tests as per the provided configuration.`, + Run: func(cmd *cobra.Command, args []string) { + + var reporter report.Reporter = report.NewConsoleReporter(os.Stderr, verboseMode) + + // Create and run the test runner + r := runner.NewRunner(&cfg, runner.RunnerOptions{ + UpdateMode: updateMode, + ContinueOnFail: continueFlag, + SnapshotsDir: snapshotsDir, + SuiteFilter: suiteFilter, + EnvFilter: envFilter, + Reporter: reporter, + }) + + result, err := r.Run(cmd.Context()) + if err != nil { + zap.L().Fatal("Test execution failed", zap.Error(err)) + } + + // Exit with code 1 if there were failures + if result.TotalFailed > 0 || result.TotalErrors > 0 { + os.Exit(1) + } + }, + } + + verifyCmd = &cobra.Command{ + Use: "verify", + Short: "Verify the end-to-end tests", + Long: `Verifies the configuration and setup for the end-to-end tests without executing them.`, + Run: func(cmd *cobra.Command, args []string) { + // Count total test cases across all suites + totalCases := 0 + for _, suite := range cfg.Suites { + totalCases += len(suite.Cases) + } + + zap.L().Info("Verification completed successfully. Configuration is valid.", + zap.Int("suites", len(cfg.Suites)), + zap.Int("cases", totalCases), + zap.Int("environments", len(cfg.Environments)), + ) + }, + } + + rootCmd = &cobra.Command{ Use: "e2e-tester", Short: "End-to-End Testing Suite for rover-ctl", Long: `A testing suite for rover-ctl commands that executes commands, captures outputs, and compares them with expected snapshots.`, + Run: func(cmd *cobra.Command, args []string) { + runCmd.Run(cmd, args) + }, + + PersistentPostRun: func(cmd *cobra.Command, args []string) { + logger.Sync() + }, + + PersistentPreRun: func(cmd *cobra.Command, args []string) { // Initialize global logger if err := logger.Initialize(logLevel, devMode); err != nil { fmt.Fprintf(os.Stderr, "Error initializing logger: %v\n", err) os.Exit(1) } - defer logger.Sync() // Setup signal handling ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + + cmd.SetContext(ctx) sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, os.Interrupt) @@ -64,37 +125,34 @@ commands, captures outputs, and compares them with expected snapshots.`, zap.L().Fatal("Error loading config", zap.Error(err)) } - var cfg config.Config if err := viper.Unmarshal(&cfg); err != nil { zap.L().Fatal("Error parsing config", zap.Error(err)) } - // Set verbose mode from flag - cfg.Verbose = verboseMode - var reporter report.Reporter = report.NewConsoleReporter(os.Stderr, verboseMode) - - // Create and run the test runner - r := runner.NewRunner(&cfg, runner.RunnerOptions{ - UpdateMode: updateMode, - ContinueOnFail: continueFlag, - SnapshotsDir: snapshotsDir, - SuiteFilter: suiteFilter, - EnvFilter: envFilter, - Reporter: reporter, - }) + // Validate configuration (initial) + if err := cfg.Validate(); err != nil { + cmd.ErrOrStderr().Write(fmt.Appendf(nil, "%v\n", err)) + os.Exit(1) + } - result, err := r.Run(ctx) - if err != nil { - zap.L().Fatal("Test execution failed", zap.Error(err)) + configDir := filepath.Dir(viper.ConfigFileUsed()) + if err := cfg.LoadSuites(configDir); err != nil { + zap.L().Fatal("Error loading test suites", zap.Error(err)) } - // Exit with code 1 if there were failures - if result.TotalFailed > 0 || result.TotalErrors > 0 { + // Validate configuration (after loading suites) + if err := cfg.Validate(); err != nil { + cmd.ErrOrStderr().Write(fmt.Appendf(nil, "%v\n", err)) os.Exit(1) } + + // Set verbose mode from flag + cfg.Verbose = verboseMode }, } +) +func main() { // Add flags rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is e2e-test-config.yaml)") rootCmd.PersistentFlags().BoolVar(&updateMode, "update", false, "update snapshots instead of comparing") @@ -108,8 +166,9 @@ commands, captures outputs, and compares them with expected snapshots.`, rootCmd.PersistentFlags().StringVar(&suiteFilter, "suite", "", "run only the specified test suite (by name)") rootCmd.PersistentFlags().StringVar(&envFilter, "env", "", "run tests only in the specified environment (by name)") - // These flags can be used together to run a specific suite in a specific environment - // Additionally, suites can specify an environment in the config file using the 'environment' field + // Add commands + rootCmd.AddCommand(runCmd) + rootCmd.AddCommand(verifyCmd) // Execute if err := rootCmd.Execute(); err != nil { diff --git a/tools/e2e-tester/pkg/command/executor.go b/tools/e2e-tester/pkg/command/executor.go index 79e68e88..51592b5c 100644 --- a/tools/e2e-tester/pkg/command/executor.go +++ b/tools/e2e-tester/pkg/command/executor.go @@ -17,6 +17,10 @@ import ( "go.uber.org/zap" ) +const ( + roverTokenEnvKey = "ROVER_TOKEN" +) + // ExecuteResult contains the result of a command execution type ExecuteResult struct { ExitCode int @@ -54,7 +58,11 @@ func (e *RoverCtlExecutor) Execute(ctx context.Context, cmdStr string, params ma // Set up environment variables if e.environment.Token != "" { - cmd.Env = append(cmd.Env, fmt.Sprintf("ROVER_TOKEN=%s", e.environment.Token)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", roverTokenEnvKey, e.environment.Token)) + } + + for _, envVar := range e.environment.Variables { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", envVar.Name, envVar.Value)) } // Capture stdout and stderr diff --git a/tools/e2e-tester/pkg/command/executor_test.go b/tools/e2e-tester/pkg/command/executor_test.go index 7972c139..068b1971 100644 --- a/tools/e2e-tester/pkg/command/executor_test.go +++ b/tools/e2e-tester/pkg/command/executor_test.go @@ -99,7 +99,7 @@ func TestExecutor_CreateSnapshot(t *testing.T) { suiteName := "test-suite" caseIndex := "0" caseName := "test-case" - snapshot := executor.CreateSnapshot(cmdStr, execResult, "", suiteName, caseIndex, caseName) + snapshot := executor.CreateSnapshot(cmdStr, execResult, environment.Name, suiteName, caseIndex, caseName) // Verify snapshot if snapshot.Id == "" { diff --git a/tools/e2e-tester/pkg/config/config.go b/tools/e2e-tester/pkg/config/config.go index 834dee1e..a5d867c0 100644 --- a/tools/e2e-tester/pkg/config/config.go +++ b/tools/e2e-tester/pkg/config/config.go @@ -6,39 +6,94 @@ package config import ( "fmt" + "path" "strings" "time" + "slices" + "github.com/spf13/viper" "go.uber.org/zap" ) +// RunPolicy defines how a test case behaves relative to prior test failures. +type RunPolicy string + +const ( + // RunPolicyNormal - test runs when prior tests passed, skipped when prior failed. + RunPolicyNormal RunPolicy = "normal" + + // RunPolicyCritical - test runs when prior passed, aborts suite on ERROR status. + RunPolicyCritical RunPolicy = "critical" + + // RunPolicyAlways - test always runs regardless of prior failures (for cleanup). + RunPolicyAlways RunPolicy = "always" +) + +// ValidRunPolicies contains all valid RunPolicy values for validation. +var ValidRunPolicies = []RunPolicy{RunPolicyNormal, RunPolicyCritical, RunPolicyAlways} + +// IsValid checks if a RunPolicy value is valid. +func (p RunPolicy) IsValid() bool { + return slices.Contains(ValidRunPolicies, p) +} + type Case struct { - Name string `mapstructure:"name"` - Description string `mapstructure:"description"` // Optional description of the test case purpose - Type string `mapstructure:"type"` // Command type: "roverctl" (default) or "snapshot" - MustPass bool `mapstructure:"must_pass"` - Command string `mapstructure:"command"` + Name string `mapstructure:"name" validate:"required"` + Description string `mapstructure:"description"` // Optional description of the test case purpose + Type string `mapstructure:"type" validate:"omitempty,oneof=roverctl snapshot"` // Command type: "roverctl" (default) or "snapshot" + RunPolicy RunPolicy `mapstructure:"run_policy" validate:"omitempty,run_policy,oneof=normal critical always"` // Execution policy: "normal" (default), "critical", "always" + Command string `mapstructure:"command" validate:"required"` Compare bool `mapstructure:"compare"` - Environment string `mapstructure:"environment"` // Optional environment to run this case in - Params map[string]any `mapstructure:"params"` // Optional type-specific parameters for future extensibility - WaitBefore time.Duration `mapstructure:"wait_before"` // Optional wait time before executing the case - WaitAfter time.Duration `mapstructure:"wait_after"` // Optional wait time after executing the case - Selector string `mapstructure:"selector"` // YAML path selector for output processing + Environment string `mapstructure:"environment"` // Optional environment to run this case in + Params map[string]any `mapstructure:"params"` // Optional type-specific parameters for future extensibility + WaitBefore time.Duration `mapstructure:"wait_before" validate:"omitempty,gte=0"` // Optional wait time before executing the case + WaitAfter time.Duration `mapstructure:"wait_after" validate:"omitempty,gte=0"` // Optional wait time after executing the case + Selector string `mapstructure:"selector"` // YAML path selector for output processing +} + +// GetRunPolicy returns the effective run policy, defaulting to "normal" if not set. +func (c *Case) GetRunPolicy() RunPolicy { + if c.RunPolicy == "" { + return RunPolicyNormal + } + return c.RunPolicy +} + +// IsCritical returns true if this case should abort the suite on ERROR. +func (c *Case) IsCritical() bool { + return c.GetRunPolicy() == RunPolicyCritical +} + +// ShouldAlwaysRun returns true if this case should run regardless of prior failures. +func (c *Case) ShouldAlwaysRun() bool { + return c.GetRunPolicy() == RunPolicyAlways +} + +// +schema:inline +// SuiteContent represents the content of a test suite and is only used for schema generation +type SuiteContent struct { + Description string `mapstructure:"description"` + Cases []*Case `mapstructure:"cases" validate:"required,min=1,dive,required"` + Environments []string `mapstructure:"environments"` } type Suite struct { - Name string `mapstructure:"name"` - Description string `mapstructure:"description"` // Optional description of the test suite purpose - Cases []*Case `mapstructure:"cases"` - Environments []string `mapstructure:"environments"` // Required list of environments to run this suite in + Name string `mapstructure:"name" validate:"required"` + Filepath string `mapstructure:"filepath"` // The path to the file where the suite is defined. Mutually exclusive with all other fields + Description string `mapstructure:"description"` // Optional description of the test suite purpose + Cases []*Case `mapstructure:"cases" validate:"required_without=Filepath,omitempty,min=1,dive,required"` // Test cases in this suite + Environments []string `mapstructure:"environments"` // Required list of environments to run this suite in } +// DeepCopy creates a deep copy of the Suite. func (s *Suite) DeepCopy() *Suite { newCases := make([]*Case, len(s.Cases)) for i, c := range s.Cases { - newCase := *c - newCases[i] = &newCase + if c != nil { + newCase := *c + newCases[i] = &newCase + } } newEnvs := make([]string, len(s.Environments)) @@ -63,48 +118,89 @@ func (s Suite) GetName() string { return s.Name } +// IsExternal checks if the suite is defined in an external file +func (s Suite) IsExternal() bool { + return s.Filepath != "" +} + +// Load loads the suite from an external file if applicable +func (s *Suite) Load(configDir string) error { + if !s.IsExternal() { + return nil + } + originalName := s.Name + + filepath := s.Filepath + if !path.IsAbs(filepath) { + filepath = path.Join(configDir, s.Filepath) + } + + zap.L().Info("Loading external suite", zap.String("file", filepath)) + v := viper.New() + v.SetConfigFile(filepath) + + if err := v.ReadInConfig(); err != nil { + return fmt.Errorf("failed to read suite file %s: %w", s.Filepath, err) + } + + zap.L().Info("Suite file loaded", zap.String("file", v.ConfigFileUsed())) + + if err := v.Unmarshal(&s); err != nil { + return fmt.Errorf("failed to unmarshal suite file %s: %w", s.Filepath, err) + } + + // Restore name from root config (authoritative) + s.Name = originalName + + // Clear the filepath after loading + s.Filepath = "" + + return nil +} + type SnapshotterConfig struct { - URL string `mapstructure:"url"` + URL string `mapstructure:"url" validate:"omitempty,url"` Binary string `mapstructure:"binary"` } type RoverCtlConfig struct { - DownloadURL string `mapstructure:"download_url"` - Binary string `mapstructure:"binary"` + DownloadURL string `mapstructure:"download_url" validate:"omitempty,url"` + Binary string `mapstructure:"binary" validate:"required"` +} + +type Variable struct { + Name string `mapstructure:"name" validate:"required"` + Value string `mapstructure:"value" validate:"required"` } type Environments struct { - Name string `mapstructure:"name"` - Token string `mapstructure:"token"` + Name string `mapstructure:"name" validate:"required"` + Token string `mapstructure:"token" validate:"required"` + Variables []Variable `mapstructure:"variables" validate:"omitempty,dive"` } +// +schema:inline type Config struct { Snapshotter SnapshotterConfig `mapstructure:"snapshotter"` - RoverCtl RoverCtlConfig `mapstructure:"roverctl"` - Environments []Environments `mapstructure:"environments"` - Suites []Suite `mapstructure:"suites"` + RoverCtl RoverCtlConfig `mapstructure:"roverctl" validate:"required"` + Environments []Environments `mapstructure:"environments" validate:"required,min=1,dive"` + Suites []Suite `mapstructure:"suites" validate:"required,min=1,dive"` Verbose bool `mapstructure:"verbose"` } -// Validate checks if the config is valid +// Validate checks if the config is valid using struct validation tags func (c *Config) Validate() error { - // Validate suites - if len(c.Suites) == 0 { - return fmt.Errorf("at least one suite must be specified") - } - - // Validate environments - if len(c.Environments) == 0 { - return fmt.Errorf("at least one environment must be specified") - } + return ValidateConfig(c) +} - // Ensure at least one case per suite and at least one environment per suite - for _, suite := range c.Suites { - if len(suite.Cases) == 0 { - return fmt.Errorf("suite %s must have at least one case", suite.Name) +func (c *Config) LoadSuites(configDir string) error { + for i := range c.Suites { + if c.Suites[i].IsExternal() { + if err := c.Suites[i].Load(configDir); err != nil { + return err + } } } - return nil } diff --git a/tools/e2e-tester/pkg/config/testdata/invalid-external-suite.yaml b/tools/e2e-tester/pkg/config/testdata/invalid-external-suite.yaml new file mode 100644 index 00000000..9011b1dd --- /dev/null +++ b/tools/e2e-tester/pkg/config/testdata/invalid-external-suite.yaml @@ -0,0 +1,8 @@ +# Copyright 2025 Deutsche Telekom IT GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +description: "Invalid external test suite - missing cases" +environments: + - "test-env" +cases: [] diff --git a/tools/e2e-tester/pkg/config/testdata/valid-external-suite.yaml b/tools/e2e-tester/pkg/config/testdata/valid-external-suite.yaml new file mode 100644 index 00000000..94df1873 --- /dev/null +++ b/tools/e2e-tester/pkg/config/testdata/valid-external-suite.yaml @@ -0,0 +1,14 @@ +# Copyright 2025 Deutsche Telekom IT GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +description: "External test suite" +environments: + - "test-env" +cases: + - name: "external-case-1" + command: "--version" + compare: true + - name: "external-case-2" + command: "--help" + compare: true diff --git a/tools/e2e-tester/pkg/config/validator.go b/tools/e2e-tester/pkg/config/validator.go new file mode 100644 index 00000000..219eea6a --- /dev/null +++ b/tools/e2e-tester/pkg/config/validator.go @@ -0,0 +1,213 @@ +// Copyright 2025 Deutsche Telekom IT GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "fmt" + "reflect" + "strings" + + "github.com/go-playground/validator/v10" +) + +var validate *validator.Validate + +func init() { + validate = validator.New(validator.WithRequiredStructEnabled()) + + // Register a tag name function to use mapstructure tags for field names + validate.RegisterTagNameFunc(func(fld reflect.StructField) string { + name := strings.SplitN(fld.Tag.Get("mapstructure"), ",", 2)[0] + if name == "-" { + return "" + } + return name + }) + + // Register custom field-level validators - panic on failure since this is initialization + if err := validate.RegisterValidation("run_policy", validateRunPolicy); err != nil { + panic(fmt.Sprintf("failed to register run_policy validator: %v", err)) + } + + // Register struct-level validators + validate.RegisterStructValidation(validateSnapshotterConfig, SnapshotterConfig{}) + validate.RegisterStructValidation(validateConfig, Config{}) + validate.RegisterStructValidation(validateSuite, Suite{}) +} + +// ValidateConfig validates the configuration using struct tags and custom validators. +// It returns a user-friendly error message aggregating all validation failures, +// or nil if the configuration is valid. +func ValidateConfig(c *Config) error { + err := validate.Struct(c) + if err != nil { + if validationErrors, ok := err.(validator.ValidationErrors); ok { + return formatValidationErrors(validationErrors) + } + return fmt.Errorf("validation error: %w", err) + } + return nil +} + +// validateRunPolicy validates RunPolicy enum values. +func validateRunPolicy(fl validator.FieldLevel) bool { + policy := RunPolicy(fl.Field().String()) + if policy == "" { + return true // Empty defaults to "normal" at runtime + } + return policy.IsValid() +} + +// validateSnapshotterConfig ensures at least URL or Binary is provided. +func validateSnapshotterConfig(sl validator.StructLevel) { + cfg, ok := sl.Current().Interface().(SnapshotterConfig) + if !ok { + return // Shouldn't happen, but be defensive + } + if cfg.URL == "" && cfg.Binary == "" { + sl.ReportError(cfg.URL, "Snapshotter", "snapshotter", "url_or_binary_required", "") + } +} + +// validateConfig performs all Config-level validations including +// cross-reference checks and uniqueness constraints. +func validateConfig(sl validator.StructLevel) { + cfg, ok := sl.Current().Interface().(Config) + if !ok { + return // Shouldn't happen, but be defensive + } + + // Validate environment cross-references + validEnvs := make(map[string]bool) + for _, env := range cfg.Environments { + validEnvs[env.Name] = true + } + + // Check suite-level environment references + for i, suite := range cfg.Suites { + for j, envName := range suite.Environments { + if envName != "" && !validEnvs[envName] { + sl.ReportError( + cfg.Suites[i].Environments[j], + fmt.Sprintf("Suites[%d].Environments[%d]", i, j), + "environments", + "env_reference", + envName, + ) + } + } + + // Check case-level environment overrides + for j, c := range suite.Cases { + if c != nil && c.Environment != "" && !validEnvs[c.Environment] { + sl.ReportError( + c.Environment, + fmt.Sprintf("Suites[%d].Cases[%d].Environment", i, j), + "environment", + "env_reference", + c.Environment, + ) + } + } + } + + // Check for duplicate environment names + envNames := make(map[string]bool) + for i, env := range cfg.Environments { + if env.Name != "" { + if envNames[env.Name] { + sl.ReportError( + cfg.Environments[i].Name, + fmt.Sprintf("Environments[%d].Name", i), + "name", + "unique_env_name", + env.Name, + ) + } + envNames[env.Name] = true + } + } + + // Check for duplicate suite names + suiteNames := make(map[string]bool) + for i, suite := range cfg.Suites { + if suite.Name != "" { + if suiteNames[suite.Name] { + sl.ReportError( + cfg.Suites[i].Name, + fmt.Sprintf("Suites[%d].Name", i), + "name", + "unique_suite_name", + suite.Name, + ) + } + suiteNames[suite.Name] = true + } + } +} + +func validateSuite(sl validator.StructLevel) { + suite, ok := sl.Current().Interface().(Suite) + if !ok { + return + } + + hasFilepath := suite.Filepath != "" + hasCases := len(suite.Cases) > 0 + + if hasFilepath && hasCases { + sl.ReportError(suite.Filepath, "filepath", "Filepath", + "filepath_cases_exclusive", "") + } +} + +// formatValidationErrors converts validator.ValidationErrors to user-friendly messages +func formatValidationErrors(errs validator.ValidationErrors) error { + var messages []string + for _, e := range errs { + messages = append(messages, formatFieldError(e)) + } + return fmt.Errorf("configuration validation failed:\n - %s", + strings.Join(messages, "\n - ")) +} + +func formatFieldError(e validator.FieldError) string { + field := e.Namespace() + // Remove "Config." prefix for cleaner output + field = strings.TrimPrefix(field, "Config.") + + switch e.Tag() { + case "required": + return fmt.Sprintf("%s is required", field) + case "min": + return fmt.Sprintf("%s must have at least %s item(s)", field, e.Param()) + case "oneof": + return fmt.Sprintf("%s must be one of: %s (got: '%v')", field, e.Param(), e.Value()) + case "unique": + return fmt.Sprintf("%s must have unique '%s' values", field, e.Param()) + case "run_policy": + validValues := make([]string, len(ValidRunPolicies)) + for i, p := range ValidRunPolicies { + validValues[i] = string(p) + } + return fmt.Sprintf("%s must be one of: %s (got: '%v')", field, strings.Join(validValues, ", "), e.Value()) + case "url_or_binary_required": + return "Snapshotter: either 'url' or 'binary' must be specified" + case "url": + return fmt.Sprintf("%s must be a valid URL (got: '%v')", field, e.Value()) + case "env_reference": + return fmt.Sprintf("%s references unknown environment '%s'", field, e.Param()) + case "unique_env_name": + return fmt.Sprintf("Environments must have unique names (duplicate: '%s')", e.Param()) + case "unique_suite_name": + return fmt.Sprintf("Suites must have unique names (duplicate: '%s')", e.Param()) + case "gte": + return fmt.Sprintf("%s must be >= %s", field, e.Param()) + case "filepath_cases_exclusive": + return fmt.Sprintf("%s: 'filepath' and 'cases' are mutually exclusive", field) + default: + return fmt.Sprintf("%s: validation failed on '%s'", field, e.Tag()) + } +} diff --git a/tools/e2e-tester/pkg/config/validator_test.go b/tools/e2e-tester/pkg/config/validator_test.go new file mode 100644 index 00000000..aad2c8bc --- /dev/null +++ b/tools/e2e-tester/pkg/config/validator_test.go @@ -0,0 +1,991 @@ +// Copyright 2025 Deutsche Telekom IT GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "strings" + "testing" + "time" +) + +func TestValidConfig(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{ + Binary: "/usr/local/bin/snapshotter", + }, + RoverCtl: RoverCtlConfig{ + Binary: "roverctl", + }, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Environments: []string{"test-env"}, + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + if err := cfg.Validate(); err != nil { + t.Errorf("expected valid config, got error: %v", err) + } +} + +func TestMissingRoverCtlBinary(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing RoverCtl.Binary") + } + if !strings.Contains(err.Error(), "roverctl") && !strings.Contains(err.Error(), "binary") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestMissingSnapshotter(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{}, // Neither URL nor Binary + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing snapshotter config") + } + if !strings.Contains(err.Error(), "Snapshotter") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestSnapshotterWithURL(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{URL: "http://localhost:8080"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + if err := cfg.Validate(); err != nil { + t.Errorf("expected valid config with snapshotter URL, got error: %v", err) + } +} + +func TestInvalidCaseType(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version", Type: "invalid-type"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for invalid case type") + } + if !strings.Contains(err.Error(), "must be one of: roverctl snapshot") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestValidCaseTypes(t *testing.T) { + types := []string{"roverctl", "snapshot", ""} + + for _, typ := range types { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version", Type: typ}, + }, + }, + }, + } + + if err := cfg.Validate(); err != nil { + t.Errorf("expected valid case type '%s', got error: %v", typ, err) + } + } +} + +func TestInvalidRunPolicy(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version", RunPolicy: "invalid"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for invalid run policy") + } + if !strings.Contains(err.Error(), "run_policy") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestValidRunPolicies(t *testing.T) { + policies := []RunPolicy{RunPolicyNormal, RunPolicyCritical, RunPolicyAlways, ""} + + for _, policy := range policies { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version", RunPolicy: policy}, + }, + }, + }, + } + + if err := cfg.Validate(); err != nil { + t.Errorf("expected valid run_policy '%s', got error: %v", policy, err) + } + } +} + +func TestEnvironmentCrossReference(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "env-a", Token: "token-a"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Environments: []string{"nonexistent-env"}, + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for unknown environment reference") + } + if !strings.Contains(err.Error(), "nonexistent-env") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestCaseEnvironmentCrossReference(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "env-a", Token: "token-a"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version", Environment: "unknown-env"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for unknown case environment reference") + } + if !strings.Contains(err.Error(), "unknown-env") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestDuplicateEnvironmentNames(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "duplicate", Token: "token-1"}, + {Name: "duplicate", Token: "token-2"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for duplicate environment names") + } + if !strings.Contains(err.Error(), "unique") && !strings.Contains(err.Error(), "duplicate") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestDuplicateSuiteNames(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "duplicate", + Cases: []*Case{ + {Name: "test-case-1", Command: "--version"}, + }, + }, + { + Name: "duplicate", + Cases: []*Case{ + {Name: "test-case-2", Command: "--help"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for duplicate suite names") + } + if !strings.Contains(err.Error(), "unique") && !strings.Contains(err.Error(), "duplicate") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestMissingCaseName(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Command: "--version"}, // Missing Name + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing case name") + } + if !strings.Contains(err.Error(), "name is required") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestMissingCommand(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case"}, // Missing Command + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing command") + } + if !strings.Contains(err.Error(), "command is required") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestEmptySuites(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{{Name: "test-env", Token: "test-token"}}, + Suites: []Suite{}, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for empty suites") + } + if !strings.Contains(err.Error(), "at least 1") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestEmptyCases(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{}, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for empty cases") + } + if !strings.Contains(err.Error(), "at least 1") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestNegativeWaitDuration(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + { + Name: "test-case", + Command: "--version", + WaitBefore: -1 * time.Second, + }, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for negative wait duration") + } + if !strings.Contains(err.Error(), "wait_before") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestAggregatedErrors(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{}, // Missing URL and Binary + RoverCtl: RoverCtlConfig{}, // Missing Binary + Environments: []Environments{ + {Name: "test-env"}, // Missing Token + }, + Suites: []Suite{ + { + Name: "", // Missing Name + Environments: []string{"nonexistent"}, // Invalid reference + Cases: []*Case{{Type: "invalid-type"}}, // Invalid type, missing name & command + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected multiple validation errors") + } + + errStr := err.Error() + // Check that multiple errors are reported + if !strings.Contains(errStr, "configuration validation failed:") { + t.Errorf("expected aggregated error format, got: %v", err) + } + // Count bullet points (error items) + bulletCount := strings.Count(errStr, "\n - ") + if bulletCount < 3 { + t.Errorf("expected at least 3 errors, got %d in: %v", bulletCount, err) + } +} + +func TestMissingSuiteName(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "", // Missing Name + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing suite name") + } + if !strings.Contains(err.Error(), "name is required") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestMissingEnvironmentName(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "", Token: "test-token"}, // Missing Name + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing environment name") + } + if !strings.Contains(err.Error(), "name is required") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestMissingEnvironmentToken(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: ""}, // Missing Token + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing environment token") + } + if !strings.Contains(err.Error(), "token is required") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestEmptyEnvironments(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{}, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for empty environments") + } + if !strings.Contains(err.Error(), "at least 1") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestVariableValidation(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + { + Name: "test-env", + Token: "test-token", + Variables: []Variable{ + {Name: "", Value: "value"}, // Missing variable name + }, + }, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for missing variable name") + } + if !strings.Contains(err.Error(), "name is required") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestValidConfigWithAllFields(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{ + URL: "http://localhost:8080", + Binary: "snapshotter", + }, + RoverCtl: RoverCtlConfig{ + Binary: "roverctl", + DownloadURL: "https://example.com/roverctl", + }, + Environments: []Environments{ + { + Name: "env-a", + Token: "token-a", + Variables: []Variable{ + {Name: "VAR1", Value: "value1"}, + {Name: "VAR2", Value: "value2"}, + }, + }, + { + Name: "env-b", + Token: "token-b", + }, + }, + Suites: []Suite{ + { + Name: "suite-1", + Description: "Test suite 1", + Environments: []string{"env-a", "env-b"}, + Cases: []*Case{ + { + Name: "case-1", + Description: "Test case 1", + Type: "roverctl", + RunPolicy: RunPolicyCritical, + Command: "--version", + Compare: true, + WaitBefore: 5 * time.Second, + WaitAfter: 2 * time.Second, + Selector: "$.version", + }, + { + Name: "case-2", + Type: "snapshot", + RunPolicy: RunPolicyAlways, + Command: "snap --source test", + Environment: "env-a", + }, + }, + }, + }, + Verbose: true, + } + + if err := cfg.Validate(); err != nil { + t.Errorf("expected valid config with all fields, got error: %v", err) + } +} + +func TestValidateNilConfig(t *testing.T) { + err := ValidateConfig(nil) + if err == nil { + t.Fatal("expected error for nil config") + } +} + +func TestInvalidSnapshotterURL(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{URL: "not-a-valid-url"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for invalid snapshotter URL") + } + if !strings.Contains(err.Error(), "url") || !strings.Contains(err.Error(), "URL") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestInvalidRoverCtlDownloadURL(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl", DownloadURL: "invalid-url"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "test-suite", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for invalid RoverCtl DownloadURL") + } + if !strings.Contains(err.Error(), "url") || !strings.Contains(err.Error(), "URL") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestDeepCopyWithNilCases(t *testing.T) { + suite := &Suite{ + Name: "test-suite", + Description: "Test suite with nil cases", + Cases: []*Case{nil, {Name: "valid-case", Command: "--version"}, nil}, + Environments: []string{"env-a"}, + } + + // Should not panic + copied := suite.DeepCopy() + + if copied.Name != suite.Name { + t.Errorf("expected name %s, got %s", suite.Name, copied.Name) + } + if len(copied.Cases) != len(suite.Cases) { + t.Errorf("expected %d cases, got %d", len(suite.Cases), len(copied.Cases)) + } + if copied.Cases[0] != nil { + t.Error("expected first case to be nil") + } + if copied.Cases[1] == nil || copied.Cases[1].Name != "valid-case" { + t.Error("expected second case to be valid") + } + if copied.Cases[2] != nil { + t.Error("expected third case to be nil") + } +} + +// Tests for external suite files feature + +func TestSuiteIsExternal(t *testing.T) { + tests := []struct { + name string + suite Suite + expected bool + }{ + { + name: "Suite with filepath is external", + suite: Suite{Name: "external-suite", Filepath: "./suites/test.yaml"}, + expected: true, + }, + { + name: "Suite without filepath is not external", + suite: Suite{Name: "inline-suite", Cases: []*Case{{Name: "case-1", Command: "--version"}}}, + expected: false, + }, + { + name: "Suite with empty filepath is not external", + suite: Suite{Name: "inline-suite", Filepath: "", Cases: []*Case{{Name: "case-1", Command: "--version"}}}, + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if tc.suite.IsExternal() != tc.expected { + t.Errorf("IsExternal() = %v, want %v", tc.suite.IsExternal(), tc.expected) + } + }) + } +} + +func TestSuiteFilepathAndCasesExclusive(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "invalid-suite", + Filepath: "./suites/test.yaml", + Cases: []*Case{ + {Name: "test-case", Command: "--version"}, + }, + }, + }, + } + + err := cfg.Validate() + if err == nil { + t.Fatal("expected error for suite with both filepath and cases") + } + if !strings.Contains(err.Error(), "filepath") || !strings.Contains(err.Error(), "cases") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestSuiteFilepathOnly(t *testing.T) { + // Suite with only filepath should be valid (cases not required when filepath is set) + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "external-suite", + Filepath: "./suites/test.yaml", + }, + }, + } + + err := cfg.Validate() + if err != nil { + t.Errorf("expected valid config with filepath only, got error: %v", err) + } +} + +func TestSuiteLoadExternalFile(t *testing.T) { + suite := Suite{ + Name: "my-external-suite", + Filepath: "testdata/valid-external-suite.yaml", + } + + // Load from the testdata directory + err := suite.Load(".") + if err != nil { + t.Fatalf("Load() failed: %v", err) + } + + // Verify the suite was loaded correctly + if suite.Name != "my-external-suite" { + t.Errorf("Name should be preserved from root config, got %s", suite.Name) + } + if suite.Filepath != "" { + t.Errorf("Filepath should be cleared after load, got %s", suite.Filepath) + } + if suite.Description != "External test suite" { + t.Errorf("Description = %q, want %q", suite.Description, "External test suite") + } + if len(suite.Cases) != 2 { + t.Fatalf("expected 2 cases, got %d", len(suite.Cases)) + } + if suite.Cases[0].Name != "external-case-1" { + t.Errorf("Cases[0].Name = %s, want external-case-1", suite.Cases[0].Name) + } + if len(suite.Environments) != 1 || suite.Environments[0] != "test-env" { + t.Errorf("Environments = %v, want [test-env]", suite.Environments) + } +} + +func TestSuiteLoadNoFilepath(t *testing.T) { + suite := Suite{ + Name: "inline-suite", + Cases: []*Case{{Name: "case-1", Command: "--version"}}, + } + + // Load should be a no-op when no filepath is set + err := suite.Load(".") + if err != nil { + t.Fatalf("Load() should succeed for suite without filepath: %v", err) + } + + // Suite should remain unchanged + if len(suite.Cases) != 1 { + t.Errorf("expected 1 case, got %d", len(suite.Cases)) + } +} + +func TestSuiteLoadNonExistentFile(t *testing.T) { + suite := Suite{ + Name: "missing-suite", + Filepath: "testdata/non-existent.yaml", + } + + err := suite.Load(".") + if err == nil { + t.Fatal("Load() should fail for non-existent file") + } + if !strings.Contains(err.Error(), "failed to read suite file") { + t.Errorf("unexpected error message: %v", err) + } +} + +func TestSuiteLoadAbsolutePath(t *testing.T) { + // Get absolute path to testdata + suite := Suite{ + Name: "absolute-path-suite", + Filepath: "testdata/valid-external-suite.yaml", + } + + // When configDir is empty, relative path should still work from current dir + err := suite.Load(".") + if err != nil { + t.Fatalf("Load() failed: %v", err) + } + if len(suite.Cases) != 2 { + t.Errorf("expected 2 cases, got %d", len(suite.Cases)) + } +} + +func TestConfigLoadSuites(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "external-suite", + Filepath: "testdata/valid-external-suite.yaml", + }, + { + Name: "inline-suite", + Cases: []*Case{ + {Name: "inline-case", Command: "--help"}, + }, + }, + }, + } + + err := cfg.LoadSuites(".") + if err != nil { + t.Fatalf("LoadSuites() failed: %v", err) + } + + // Verify external suite was loaded + if len(cfg.Suites[0].Cases) != 2 { + t.Errorf("external suite should have 2 cases, got %d", len(cfg.Suites[0].Cases)) + } + if cfg.Suites[0].Filepath != "" { + t.Errorf("Filepath should be cleared after load") + } + + // Verify inline suite remains unchanged + if len(cfg.Suites[1].Cases) != 1 { + t.Errorf("inline suite should have 1 case, got %d", len(cfg.Suites[1].Cases)) + } +} + +func TestConfigLoadSuitesWithError(t *testing.T) { + cfg := &Config{ + Snapshotter: SnapshotterConfig{Binary: "snapshotter"}, + RoverCtl: RoverCtlConfig{Binary: "roverctl"}, + Environments: []Environments{ + {Name: "test-env", Token: "test-token"}, + }, + Suites: []Suite{ + { + Name: "missing-suite", + Filepath: "testdata/non-existent.yaml", + }, + }, + } + + err := cfg.LoadSuites(".") + if err == nil { + t.Fatal("LoadSuites() should fail for non-existent file") + } +} diff --git a/tools/e2e-tester/pkg/environment/manager.go b/tools/e2e-tester/pkg/environment/manager.go index 94f74d72..3b250a59 100644 --- a/tools/e2e-tester/pkg/environment/manager.go +++ b/tools/e2e-tester/pkg/environment/manager.go @@ -14,10 +14,15 @@ import ( "go.uber.org/zap" ) +const ( + envPrefix = "env://" +) + type EnvironmentManager interface { GetEnvironment(name string) (*config.Environments, error) GetAllEnvironments() []config.Environments ResolveTokenFromEnv() error + ResolveVariablesFromEnv() error SetupTestEnvironment(environment string) (*config.Environments, error) GetExecutor(envName string) (*command.Executor, error) ValidateEnvironments(envNames []string) error @@ -57,12 +62,31 @@ func (m *Manager) GetAllEnvironments() []config.Environments { return m.environments } +// ResolveVariablesFromEnv resolves variable values that reference environment variables +func (m *Manager) ResolveVariablesFromEnv() error { + for i := range m.environments { + env := &m.environments[i] + for j := range env.Variables { + variable := &env.Variables[j] + if strings.HasPrefix(variable.Value, envPrefix) { + envVarName := strings.TrimPrefix(variable.Value, envPrefix) + envValue := os.Getenv(envVarName) + if envValue == "" { + return fmt.Errorf("environment variable not set: %s", envVarName) + } + variable.Value = envValue + } + } + } + return nil +} + // ResolveTokenFromEnv resolves token values that reference environment variables func (m *Manager) ResolveTokenFromEnv() error { for i := range m.environments { env := &m.environments[i] - if strings.HasPrefix(env.Token, "env:") { - envVarName := strings.TrimPrefix(env.Token, "env:") + if strings.HasPrefix(env.Token, envPrefix) { + envVarName := strings.TrimPrefix(env.Token, envPrefix) envValue := os.Getenv(envVarName) if envValue == "" { return fmt.Errorf("environment variable not set: %s", envVarName) @@ -82,8 +106,8 @@ func (m *Manager) SetupTestEnvironment(environment string) (*config.Environments } // Ensure token is resolved - if strings.HasPrefix(env.Token, "env:") { - envVarName := strings.TrimPrefix(env.Token, "env:") + if strings.HasPrefix(env.Token, envPrefix) { + envVarName := strings.TrimPrefix(env.Token, envPrefix) envValue := os.Getenv(envVarName) if envValue == "" { return nil, fmt.Errorf("environment variable not set: %s", envVarName) diff --git a/tools/e2e-tester/pkg/obfuscator/patterns.go b/tools/e2e-tester/pkg/obfuscator/patterns.go index f89ae37b..8f331265 100644 --- a/tools/e2e-tester/pkg/obfuscator/patterns.go +++ b/tools/e2e-tester/pkg/obfuscator/patterns.go @@ -52,6 +52,19 @@ var ( Pattern: `irisClientSecret: .*`, Replace: "irisClientSecret: OBFUSCATED", }, + { + Pattern: `trd_.+`, + Replace: "trd_OBFUSCATED", + }, + + { + Pattern: "uid.*", + Replace: "uid_OBFUSCATED", + }, + { + Pattern: "traceId.*", + Replace: "traceId_OBFUSCATED", + }, } ) diff --git a/tools/e2e-tester/pkg/obfuscator/patterns_test.go b/tools/e2e-tester/pkg/obfuscator/patterns_test.go index 9a7b4e85..cedc097a 100644 --- a/tools/e2e-tester/pkg/obfuscator/patterns_test.go +++ b/tools/e2e-tester/pkg/obfuscator/patterns_test.go @@ -47,7 +47,7 @@ func TestObfuscation(t *testing.T) { { name: "Token generation", input: "Token: 1761134582 (generated 3 day(s) ago)", - expected: "Token: 1761134582 (generated X day(s) ago)", + expected: "Token: 1761134582 (generated X ago)", }, { name: "Multiple patterns", diff --git a/tools/e2e-tester/pkg/report/reporter.go b/tools/e2e-tester/pkg/report/reporter.go index 08a05cf9..75e3e1bb 100644 --- a/tools/e2e-tester/pkg/report/reporter.go +++ b/tools/e2e-tester/pkg/report/reporter.go @@ -35,7 +35,7 @@ type TestCaseResult struct { Error error ComparisonDiff string Environment string - MustPass bool + RunPolicy string // Execution policy: "normal", "critical", "always" SkipReason string // Reason for skipping this test case } @@ -126,18 +126,27 @@ func (r *ConsoleReporter) ReportTestCase(result *TestCaseResult) { fmt.Fprintf(r.output, " "+color.RedString("Error: %s\n"), result.Error) } - // If the test case has MustPass flag and is in ERROR state, provide additional context - if result.MustPass && result.Status == StatusError { - fmt.Fprintln(r.output, " "+color.RedString("Critical Error:")) - fmt.Fprintf(r.output, " %s\n", color.RedString("This test case is marked as must_pass but had an execution error")) - fmt.Fprintf(r.output, " %s\n", color.RedString("This will cause the entire test suite to abort")) - } - - // If the test case has MustPass flag and FAILED (comparison), it doesn't abort but should be highlighted - if result.MustPass && result.Status == StatusFailed { - fmt.Fprintln(r.output, " "+color.YellowString("Important Test:")) - fmt.Fprintf(r.output, " %s\n", color.YellowString("This test case is marked as must_pass")) - fmt.Fprintf(r.output, " %s\n", color.YellowString("The test will continue but the final result will be marked as failed")) + // Handle run_policy-specific messaging + switch result.RunPolicy { + case "critical": + if result.Status == StatusError { + fmt.Fprintln(r.output, " "+color.RedString("Critical Error:")) + fmt.Fprintf(r.output, " %s\n", color.RedString("This test case has run_policy: critical")) + fmt.Fprintf(r.output, " %s\n", color.RedString("Suite execution will be aborted")) + } else if result.Status == StatusFailed { + fmt.Fprintln(r.output, " "+color.YellowString("Critical Test Failed:")) + fmt.Fprintf(r.output, " %s\n", color.YellowString("This test case has run_policy: critical")) + fmt.Fprintf(r.output, " %s\n", color.YellowString("Test continues but result is marked as failed")) + } + case "always": + if result.Status == StatusSkipped { + // This shouldn't happen for always policy, but handle gracefully + fmt.Fprintln(r.output, " "+color.YellowString("Note: This test has run_policy: always")) + } + case "normal": + if result.Status == StatusSkipped && result.SkipReason != "" { + fmt.Fprintf(r.output, " "+color.YellowString("Skipped: %s\n"), result.SkipReason) + } } // If command failed with non-zero exit code, show exit code @@ -397,9 +406,12 @@ func (r *ConsoleReporter) ReportFinal(report *FinalReport) { ) } - // Add must_pass flag if relevant - if c.MustPass { - fmt.Fprintln(r.output, " "+color.RedString("⚠ Critical Test (must_pass)")) + // Add run_policy indicator if relevant + switch c.RunPolicy { + case "critical": + fmt.Fprintln(r.output, " "+color.RedString("⚠ Critical Test (run_policy: critical)")) + case "always": + fmt.Fprintln(r.output, " "+color.CyanString("◆ Always-run Test (run_policy: always)")) } // Add a separator between test entries diff --git a/tools/e2e-tester/pkg/runner/runner.go b/tools/e2e-tester/pkg/runner/runner.go index 795d1941..e491e6d1 100644 --- a/tools/e2e-tester/pkg/runner/runner.go +++ b/tools/e2e-tester/pkg/runner/runner.go @@ -252,6 +252,12 @@ func (r *Runner) Run(ctx context.Context) (*report.FinalReport, error) { return nil, fmt.Errorf("failed to resolve environment tokens: %w", err) } + if err := r.envManager.ResolveVariablesFromEnv(); err != nil { + zap.L().Error("Failed to resolve environment variables", + zap.Error(err)) + return nil, fmt.Errorf("failed to resolve environment variables: %w", err) + } + // Build all suite runners zap.L().Debug("Building suite runners") suiteRunners, err := r.buildSuites() @@ -315,16 +321,17 @@ func (r *Runner) Run(ctx context.Context) (*report.FinalReport, error) { // Check if we should continue on failure if !r.continueOnFail { - // Check if any must-pass test cases failed with error status + // Check if any critical-policy test cases had ERROR status hasCriticalFailure := false for _, tc := range result.Cases { - if tc.MustPass && tc.Status == report.StatusError { + if tc.RunPolicy == "critical" && tc.Status == report.StatusError { hasCriticalFailure = true criticalFailures++ zap.L().Warn("Critical test case failed", zap.String("suite", result.Name), zap.String("case", tc.Name), - zap.String("environment", tc.Environment)) + zap.String("environment", tc.Environment), + zap.String("run_policy", tc.RunPolicy)) break } } diff --git a/tools/e2e-tester/pkg/runner/suite.go b/tools/e2e-tester/pkg/runner/suite.go index 4f22bcde..300ba924 100644 --- a/tools/e2e-tester/pkg/runner/suite.go +++ b/tools/e2e-tester/pkg/runner/suite.go @@ -6,6 +6,7 @@ package runner import ( "context" + "errors" "fmt" "maps" "slices" @@ -17,6 +18,7 @@ import ( "github.com/telekom/controlplane/tools/e2e-tester/pkg/environment" "github.com/telekom/controlplane/tools/e2e-tester/pkg/report" "github.com/telekom/controlplane/tools/e2e-tester/pkg/snapshot" + "github.com/telekom/controlplane/tools/snapshotter/pkg/store" "go.uber.org/zap" ) @@ -32,6 +34,7 @@ type SuiteRunner struct { config *config.Config // Full configuration roverCtlConfig config.RoverCtlConfig executors map[string]command.CommandExecutor + hasFailure bool // Tracks if any test has failed in this suite } // NewSuiteRunner creates a new suite runner @@ -228,23 +231,6 @@ func (r *SuiteRunner) getCommandExecutor(envName string, commandType command.Com return executor, nil } -// getExecutor returns a cached rover-ctl executor for the specified environment -// This is kept for backward compatibility -func (r *SuiteRunner) getExecutor(envName string) (*command.RoverCtlExecutor, error) { - executor, err := r.getCommandExecutor(envName, command.RoverCtlCommandType) - if err != nil { - return nil, err - } - - // Type assertion to get the concrete type - roverExecutor, ok := executor.(*command.RoverCtlExecutor) - if !ok { - return nil, fmt.Errorf("failed to convert executor to rover-ctl executor") - } - - return roverExecutor, nil -} - // Run executes all cases in the suite func (r *SuiteRunner) Run(ctx context.Context) *report.SuiteResult { startTime := time.Now() @@ -268,17 +254,47 @@ func (r *SuiteRunner) Run(ctx context.Context) *report.SuiteResult { // Execute each test case in order for i, testCase := range r.suite.Cases { + policy := testCase.GetRunPolicy() + + // Determine if this test should be skipped based on run_policy + if r.hasFailure && policy == config.RunPolicyNormal { + // Skip normal tests when suite has prior failures + caseResult := &report.TestCaseResult{ + Name: testCase.Name, + Description: testCase.Description, + Command: testCase.Command, + Environment: testCase.Environment, + RunPolicy: string(policy), + Status: report.StatusSkipped, + SkipReason: "Skipped due to prior test failure (run_policy: normal)", + } + suiteResult.Cases = append(suiteResult.Cases, caseResult) + r.reporter.ReportTestCase(caseResult) + + zap.L().Info("Skipping test case due to prior failure", + zap.String("case", testCase.Name), + zap.String("run_policy", string(policy)), + ) + continue + } + // Execute the test case caseResult := r.runCase(ctx, *testCase, i) suiteResult.Cases = append(suiteResult.Cases, caseResult) r.reporter.ReportTestCase(caseResult) - // Check if we should continue after a failure - if !r.continueOnFail && testCase.MustPass && caseResult.Status == report.StatusError { + // Track failure state for subsequent normal-policy tests + if caseResult.Status == report.StatusError || caseResult.Status == report.StatusFailed { + r.hasFailure = true + } + + // Check if we should abort suite (critical policy + ERROR status) + if !r.continueOnFail && policy == config.RunPolicyCritical && caseResult.Status == report.StatusError { zap.L().Warn("Aborting suite execution due to critical test failure", zap.String("case", testCase.Name), zap.String("suite", r.suite.Name), + zap.String("run_policy", string(policy)), ) break } @@ -304,7 +320,7 @@ func (r *SuiteRunner) runCase(ctx context.Context, c config.Case, caseIndex int) Description: c.Description, Command: c.Command, Environment: c.Environment, - MustPass: c.MustPass, + RunPolicy: string(c.GetRunPolicy()), } // Determine the command type - default to "roverctl" if not specified @@ -389,8 +405,16 @@ func (r *SuiteRunner) runCase(ctx context.Context, c config.Case, caseIndex int) } // If we're not in update mode, compare with the expected snapshot - expectedSnapshot, err := r.snapshotMgr.GetLatestSnapshot(ctx, snapshot.Id) + expectedSnapshot, err := r.snapshotMgr.GetLatestSnapshot(ctx, snapshot.ID()) if err != nil { + if errors.Is(err, store.ErrNotFound) { + zap.L().Info("no existing snapshot found, will create initial snapshot", + zap.String("id", snapshot.ID())) + } else { + result.Status = report.StatusError + result.Error = fmt.Errorf("failed to retrieve expected snapshot: %w", err) + return result + } // If the snapshot doesn't exist, it means this is the first run // and it must be created if err := r.snapshotMgr.StoreSnapshot(ctx, snapshot); err != nil { diff --git a/tools/e2e-tester/pkg/snapshot/manager.go b/tools/e2e-tester/pkg/snapshot/manager.go index 27bbdf65..5b696348 100644 --- a/tools/e2e-tester/pkg/snapshot/manager.go +++ b/tools/e2e-tester/pkg/snapshot/manager.go @@ -95,7 +95,7 @@ type ComparisonResult struct { // CompareSnapshots compares two command snapshots and returns the differences func (m *Manager) CompareSnapshots(expected, actual *CommandSnapshot) *ComparisonResult { - zap.L().Debug("Comparing snapshots", zap.String("spected", expected.ID()), zap.String("actual", actual.ID())) + zap.L().Debug("Comparing snapshots", zap.String("expected", expected.ID()), zap.String("actual", actual.ID())) // Use the diffmatcher to compare snapshots result := diffmatcher.Compare(expected, actual) diff --git a/tools/e2e-tester/schemas/config.schema.json b/tools/e2e-tester/schemas/config.schema.json new file mode 100644 index 00000000..05d47ebd --- /dev/null +++ b/tools/e2e-tester/schemas/config.schema.json @@ -0,0 +1,175 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/telekom/controlplane/refs/heads/main/tools/e2e-tester/schemas/config.schema.json", + "properties": { + "snapshotter": { + "properties": { + "url": { + "type": "string", + "format": "uri" + }, + "binary": { + "type": "string" + } + }, + "type": "object" + }, + "roverctl": { + "properties": { + "download_url": { + "type": "string", + "format": "uri" + }, + "binary": { + "type": "string" + } + }, + "type": "object", + "required": [ + "binary" + ] + }, + "environments": { + "items": { + "properties": { + "name": { + "type": "string" + }, + "token": { + "type": "string" + }, + "variables": { + "items": { + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "required": [ + "name", + "value" + ] + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "name", + "token" + ] + }, + "type": "array" + }, + "suites": { + "items": { + "properties": { + "name": { + "type": "string" + }, + "filepath": { + "type": "string", + "description": "The path to the file where the suite is defined. Mutually exclusive with all other fields" + }, + "description": { + "type": "string", + "description": "Optional description of the test suite purpose" + }, + "cases": { + "items": { + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string", + "description": "Optional description of the test case purpose" + }, + "type": { + "type": "string", + "enum": [ + "roverctl", + "snapshot" + ], + "description": "Command type: \"roverctl\" (default) or \"snapshot\"" + }, + "run_policy": { + "type": "string", + "enum": [ + "normal", + "critical", + "always" + ], + "description": "Execution policy: \"normal\" (default), \"critical\", \"always\"" + }, + "command": { + "type": "string" + }, + "compare": { + "type": "boolean" + }, + "environment": { + "type": "string", + "description": "Optional environment to run this case in" + }, + "params": { + "additionalProperties": true, + "type": "object", + "description": "Optional type-specific parameters for future extensibility" + }, + "wait_before": { + "type": "string", + "format": "duration", + "description": "Optional wait time before executing the case" + }, + "wait_after": { + "type": "string", + "format": "duration", + "description": "Optional wait time after executing the case" + }, + "selector": { + "type": "string", + "description": "YAML path selector for output processing" + } + }, + "type": "object", + "required": [ + "name", + "command" + ] + }, + "type": "array", + "description": "Test cases in this suite" + }, + "environments": { + "items": { + "type": "string" + }, + "type": "array", + "description": "Required list of environments to run this suite in" + } + }, + "type": "object", + "required": [ + "name" + ] + }, + "type": "array" + }, + "verbose": { + "type": "boolean" + } + }, + "type": "object", + "required": [ + "roverctl", + "environments", + "suites" + ], + "title": "Config", + "description": "+schema:inline" +} \ No newline at end of file diff --git a/tools/e2e-tester/schemas/config.schema.json.license b/tools/e2e-tester/schemas/config.schema.json.license new file mode 100644 index 00000000..be863cd5 --- /dev/null +++ b/tools/e2e-tester/schemas/config.schema.json.license @@ -0,0 +1,3 @@ +Copyright 2026 Deutsche Telekom IT GmbH + +SPDX-License-Identifier: Apache-2.0 diff --git a/tools/e2e-tester/schemas/suitecontent.schema.json b/tools/e2e-tester/schemas/suitecontent.schema.json new file mode 100644 index 00000000..de4422f2 --- /dev/null +++ b/tools/e2e-tester/schemas/suitecontent.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/telekom/controlplane/refs/heads/main/tools/e2e-tester/schemas/suitecontent.schema.json", + "properties": { + "description": { + "type": "string" + }, + "cases": { + "items": { + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string", + "description": "Optional description of the test case purpose" + }, + "type": { + "type": "string", + "enum": [ + "roverctl", + "snapshot" + ], + "description": "Command type: \"roverctl\" (default) or \"snapshot\"" + }, + "run_policy": { + "type": "string", + "enum": [ + "normal", + "critical", + "always" + ], + "description": "Execution policy: \"normal\" (default), \"critical\", \"always\"" + }, + "command": { + "type": "string" + }, + "compare": { + "type": "boolean" + }, + "environment": { + "type": "string", + "description": "Optional environment to run this case in" + }, + "params": { + "additionalProperties": true, + "type": "object", + "description": "Optional type-specific parameters for future extensibility" + }, + "wait_before": { + "type": "string", + "format": "duration", + "description": "Optional wait time before executing the case" + }, + "wait_after": { + "type": "string", + "format": "duration", + "description": "Optional wait time after executing the case" + }, + "selector": { + "type": "string", + "description": "YAML path selector for output processing" + } + }, + "type": "object", + "required": [ + "name", + "command" + ] + }, + "type": "array" + }, + "environments": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object", + "required": [ + "cases" + ], + "title": "SuiteContent", + "description": "+schema:inline SuiteContent represents the content of a test suite and is only used for schema generation" +} \ No newline at end of file diff --git a/tools/e2e-tester/schemas/suitecontent.schema.json.license b/tools/e2e-tester/schemas/suitecontent.schema.json.license new file mode 100644 index 00000000..be863cd5 --- /dev/null +++ b/tools/e2e-tester/schemas/suitecontent.schema.json.license @@ -0,0 +1,3 @@ +Copyright 2026 Deutsche Telekom IT GmbH + +SPDX-License-Identifier: Apache-2.0 diff --git a/tools/e2e-tester/tools/generate.go b/tools/e2e-tester/tools/generate.go new file mode 100644 index 00000000..6c98731f --- /dev/null +++ b/tools/e2e-tester/tools/generate.go @@ -0,0 +1,7 @@ +// Copyright 2026 Deutsche Telekom IT GmbH +// +// SPDX-License-Identifier: Apache-2.0 + +package tools + +//go:generate go tool github.com/ron96g/json-schema-gen --tag=mapstructure --output-dir=../schemas --schema-id=https://raw.githubusercontent.com/telekom/controlplane/refs/heads/main/tools/e2e-tester/schemas -r ../pkg/config