Merge pull request #395 from gangwgr/add-openshift-tests-extension

CNTRLPLANE-1307: set up openshift-tests-extension for cluster-openshift-controller-manager-operator and add a sanity test

Author: openshift-merge-bot[bot]

.gitignore

@@ -1,4 +1,7 @@
 /cluster-openshift-controller-manager-operator
+/cluster-openshift-controller-manager-operator-tests-ext
 .idea/
 .vscode/
 _output/
+# OpenShift tests extension generated metadata
+.openshift-tests-extension/

.openshift-tests-extension/openshift_payload_cluster-openshift-controller-manager-operator.json

@@ -0,0 +1,16 @@
+[
+  {
+    "name": "[Jira:openshift-controller-manager][sig-openshift-controller-manager] sanity test should always pass [Suite:openshift/cluster-openshift-controller-manager-operator/conformance/parallel]",
+    "labels": {},
+    "resources": {
+      "isolation": {}
+    },
+    "source": "openshift:payload:cluster-openshift-controller-manager-operator",
+    "codeLocations": [
+      "/Users/rgangwar/Downloads/backupoffice/cluster-openshift-controller-manager-operator/test/extended/main.go:8",
+      "/Users/rgangwar/Downloads/backupoffice/cluster-openshift-controller-manager-operator/test/extended/main.go:9"
+    ],
+    "lifecycle": "blocking",
+    "environmentSelector": {}
+  }
+]

Dockerfile

@@ -1,10 +1,13 @@
 FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.24-openshift-4.20 AS builder
 WORKDIR /go/src/github.com/openshift/cluster-openshift-controller-manager-operator
 COPY . .
-RUN GO_COMPLIANCE_INFO=0 make
+RUN GO_COMPLIANCE_INFO=0 make build \
+    && make tests-ext-build \
+    && gzip cluster-openshift-controller-manager-operator-tests-ext
 
 FROM registry.ci.openshift.org/ocp/4.20:base-rhel9
 COPY --from=builder /go/src/github.com/openshift/cluster-openshift-controller-manager-operator/cluster-openshift-controller-manager-operator /usr/bin/
+COPY --from=builder /go/src/github.com/openshift/cluster-openshift-controller-manager-operator/cluster-openshift-controller-manager-operator-tests-ext.gz /usr/bin/
 COPY manifests /manifests
 COPY empty-resources /manifests
 LABEL io.openshift.release.operator true

Makefile

@@ -23,3 +23,30 @@ test-e2e: GO_TEST_PACKAGES :=./test/e2e/...
 test-e2e: GO_TEST_FLAGS += -v -count=1
 test-e2e: test-unit
 .PHONY: test-e2e
+
+# -------------------------------------------------------------------
+# OpenShift Tests Extension (Cluster OpenShift Controller Manager Operator)
+# -------------------------------------------------------------------
+TESTS_EXT_BINARY := cluster-openshift-controller-manager-operator-tests-ext
+TESTS_EXT_PACKAGE := ./cmd/cluster-openshift-controller-manager-operator-tests-ext
+
+TESTS_EXT_GIT_COMMIT := $(shell git rev-parse --short HEAD)
+TESTS_EXT_BUILD_DATE := $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
+TESTS_EXT_GIT_TREE_STATE := $(shell if git diff-index --quiet HEAD --; then echo clean; else echo dirty; fi)
+
+TESTS_EXT_LDFLAGS := \
+	-X 'main.CommitFromGit=$(TESTS_EXT_GIT_COMMIT)' \
+	-X 'main.BuildDate=$(TESTS_EXT_BUILD_DATE)' \
+	-X 'main.GitTreeState=$(TESTS_EXT_GIT_TREE_STATE)'
+
+.PHONY: tests-ext-build
+tests-ext-build:
+	GOOS=$(GOOS) GOARCH=$(GOARCH) GO_COMPLIANCE_POLICY=exempt_all CGO_ENABLED=0 go build -o $(TESTS_EXT_BINARY) -ldflags "$(TESTS_EXT_LDFLAGS)" $(TESTS_EXT_PACKAGE)
+
+.PHONY: tests-ext-update
+tests-ext-update:
+	./$(TESTS_EXT_BINARY) update
+
+.PHONY: tests-ext-clean
+tests-ext-clean:
+	rm -f $(TESTS_EXT_BINARY) $(TESTS_EXT_BINARY).gz

cmd/cluster-openshift-controller-manager-operator-tests-ext/main.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/openshift-eng/openshift-tests-extension/pkg/cmd"
+	e "github.com/openshift-eng/openshift-tests-extension/pkg/extension"
+	et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
+	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
+
+	"github.com/spf13/cobra"
+
+	// The import below is necessary to ensure that the cluster openshift controller manager operator tests are registered with the extension.
+	_ "github.com/openshift/cluster-openshift-controller-manager-operator/test/extended"
+)
+
+func main() {
+	registry := e.NewRegistry()
+	ext := e.NewExtension("openshift", "payload", "cluster-openshift-controller-manager-operator")
+
+	// Suite: conformance/parallel (fast, parallel-safe)
+	ext.AddSuite(e.Suite{
+		Name:    "openshift/cluster-openshift-controller-manager-operator/conformance/parallel",
+		Parents: []string{"openshift/conformance/parallel"},
+		Qualifiers: []string{
+			`!(name.contains("[Serial]") || name.contains("[Slow]"))`,
+		},
+	})
+
+	// Suite: conformance/serial (explicitly serial tests)
+	ext.AddSuite(e.Suite{
+		Name:    "openshift/cluster-openshift-controller-manager-operator/conformance/serial",
+		Parents: []string{"openshift/conformance/serial"},
+		Qualifiers: []string{
+			`name.contains("[Serial]")`,
+		},
+	})
+
+	// Suite: optional/slow (long-running tests)
+	ext.AddSuite(e.Suite{
+		Name:    "openshift/cluster-openshift-controller-manager-operator/optional/slow",
+		Parents: []string{"openshift/optional/slow"},
+		Qualifiers: []string{
+			`name.contains("[Slow]")`,
+		},
+	})
+
+	// Suite: all (includes everything)
+	ext.AddSuite(e.Suite{
+		Name: "openshift/cluster-openshift-controller-manager-operator/all",
+	})
+
+	specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite()
+	if err != nil {
+		panic(fmt.Sprintf("couldn't build extension test specs from ginkgo: %+v", err.Error()))
+	}
+
+	// Ensure [Disruptive] tests are also [Serial] (for any future tests that might need this)
+	specs = specs.Walk(func(spec *et.ExtensionTestSpec) {
+		if strings.Contains(spec.Name, "[Disruptive]") && !strings.Contains(spec.Name, "[Serial]") {
+			spec.Name = strings.ReplaceAll(
+				spec.Name,
+				"[Disruptive]",
+				"[Serial][Disruptive]",
+			)
+		}
+	})
+
+	// Preserve original-name labels for renamed tests
+	specs = specs.Walk(func(spec *et.ExtensionTestSpec) {
+		for label := range spec.Labels {
+			if strings.HasPrefix(label, "original-name:") {
+				parts := strings.SplitN(label, "original-name:", 2)
+				if len(parts) > 1 {
+					spec.OriginalName = parts[1]
+				}
+			}
+		}
+	})
+
+	// Ignore obsolete tests
+	ext.IgnoreObsoleteTests(
+	// "[sig-openshift-controller-manager] <test name here>",
+	)
+
+	// Initialize environment before running any tests
+	specs.AddBeforeAll(func() {
+		// do stuff
+	})
+
+	ext.AddSpecs(specs)
+	registry.Register(ext)
+
+	root := &cobra.Command{
+		Long: "Cluster OpenShift Controller Manager Operator Tests Extension",
+	}
+
+	root.AddCommand(cmd.DefaultExtensionCommands(registry)...)
+
+	if err := root.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

go.mod

@@ -5,6 +5,9 @@ go 1.24.0
 require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/google/go-cmp v0.7.0
+	github.com/onsi/ginkgo/v2 v2.21.0
+	github.com/onsi/gomega v1.35.1
+	github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292
 	github.com/openshift/api v0.0.0-20250710082954-674ad74beffc
 	github.com/openshift/build-machinery-go v0.0.0-20250602125535-1b6d00b8c37c
 	github.com/openshift/client-go v0.0.0-20250710075018-396b36f983ee
@@ -40,6 +43,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.1.3 // indirect
@@ -93,6 +97,7 @@ require (
 	golang.org/x/term v0.33.0 // indirect
 	golang.org/x/text v0.27.0 // indirect
 	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/tools v0.34.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
 	google.golang.org/grpc v1.68.1 // indirect
@@ -114,3 +119,5 @@ require (
 	sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect
 	sigs.k8s.io/yaml v1.5.0 // indirect
 )
+
+replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12

go.sum

@@ -61,7 +61,6 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En
 github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
 github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
-github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
 github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
@@ -135,10 +134,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
-github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
 github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292 h1:3athg6KQ+TaNfW4BWZDlGFt1ImSZEJWgzXtPC1VPITI=
+github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M=
 github.com/openshift/api v0.0.0-20250710082954-674ad74beffc h1:DpweDMwuEuZKVUmYeqj3kBKdt5q8lGwSoIqpnLb0A9M=
 github.com/openshift/api v0.0.0-20250710082954-674ad74beffc/go.mod h1:SPLf21TYPipzCO67BURkCfK6dcIIxx0oNRVWaOyRcXM=
 github.com/openshift/build-machinery-go v0.0.0-20250602125535-1b6d00b8c37c h1:gJvhduWIrpzoUTwrJjjeul+hGETKkhRhEZosBg/X3Hg=
@@ -147,6 +146,8 @@ github.com/openshift/client-go v0.0.0-20250710075018-396b36f983ee h1:tOtrrxfDEW8
 github.com/openshift/client-go v0.0.0-20250710075018-396b36f983ee/go.mod h1:zhRiYyNMk89llof2qEuGPWPD+joQPhCRUc2IK0SB510=
 github.com/openshift/library-go v0.0.0-20250711143941-47604345e7ea h1:0BNis5UGo5Z7J9GtRY1nw/pt8hWxIZqvfqnqH3eV5cs=
 github.com/openshift/library-go v0.0.0-20250711143941-47604345e7ea/go.mod h1:tptKNust9MdRI0p90DoBSPHIrBa9oh+Rok59tF0vT8c=
+github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 h1:AKx/w1qpS8We43bsRgf8Nll3CGlDHpr/WAXvuedTNZI=
+github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

test/extended/README.md

@@ -0,0 +1,153 @@
+# OpenShift Controller Manager Operator Tests Extension
+========================================================
+
+This repository contains the tests for the OpenShift Controller Manager Operator for OpenShift.
+These tests run against OpenShift clusters and are meant to be used in the OpenShift CI/CD pipeline.
+They use the framework: https://github.com/openshift-eng/openshift-tests-extension
+
+## How to Run the Tests Locally
+
+| Command | Description |
+|---------|-------------|
+| `make tests-ext-build` | Builds the test extension binary. |
+| `./cluster-openshift-controller-manager-operator-tests-ext list` | Lists all available test cases. |
+| `./cluster-openshift-controller-manager-operator-tests-ext run-suite <suite-name>` | Runs a test suite. e.g., `openshift/cluster-openshift-controller-manager-operator/conformance/parallel` |
+| `./cluster-openshift-controller-manager-operator-tests-ext run-test <test-name>` | Runs one specific test. |
+
+The tests can be run locally using the `cluster-openshift-controller-manager-operator-tests-ext` binary against an OpenShift cluster.
+Use the environment variable `KUBECONFIG` to point to your cluster configuration file such as:
+
+```shell
+export KUBECONFIG=path/to/kubeconfig
+./cluster-openshift-controller-manager-operator-tests-ext run-test <test-name>
+```
+
+### Local Test using OCP
+
+1. Use the `Cluster Bot` to create an OpenShift cluster.
+
+**Example:**
+```shell
+launch 4.20 gcp,techpreview
+```
+
+2. Set the `KUBECONFIG` environment variable to point to your OpenShift cluster configuration file.
+
+**Example:**
+```shell
+mv ~/Downloads/cluster-bot-2025-08-06-082741.kubeconfig ~/.kube/cluster-bot.kubeconfig
+export KUBECONFIG=~/.kube/cluster-bot.kubeconfig
+```
+
+3. Run the tests using the `cluster-openshift-controller-manager-operator-tests-ext` binary.
+
+**Example:**
+```shell
+./cluster-openshift-controller-manager-operator-tests-ext run-suite openshift/cluster-openshift-controller-manager-operator/all
+```
+
+### Running with JUnit Output
+
+To generate JUnit XML reports for CI integration:
+
+```shell
+./cluster-openshift-controller-manager-operator-tests-ext run-suite openshift/cluster-openshift-controller-manager-operator/conformance/parallel --junit-path $(ARTIFACT_DIR)/junit_$(shell date +%Y%m%d-%H%M%S).xml
+```
+
+This generates both OTE framework and Ginkgo JUnit XML reports that can be integrated into CI systems.
+
+## Available Test Suites
+
+| Suite Name | Description |
+|------------|-------------|
+| `openshift/cluster-openshift-controller-manager-operator/conformance/parallel` | Parallel conformance tests |
+| `openshift/cluster-openshift-controller-manager-operator/conformance/serial` | Serial conformance tests |
+| `openshift/cluster-openshift-controller-manager-operator/optional/slow` | Optional slow tests |
+| `openshift/cluster-openshift-controller-manager-operator/all` | All tests |
+
+## CI/CD Integration
+
+The tests are integrated into the OpenShift CI/CD pipeline through the release configuration.
+The CI configuration runs the OTE binary and generates JUnit reports for test result tracking.
+
+**Example CI step:**
+```yaml
+- as: tests-extension
+  commands: |
+    echo "Build binary cluster-openshift-controller-manager-operator-tests-ext"
+    make tests-ext-build
+    echo "Running ./cluster-openshift-controller-manager-operator-tests-ext with sanity test"
+    ./cluster-openshift-controller-manager-operator-tests-ext run-suite \
+      "openshift/cluster-openshift-controller-manager-operator/conformance/parallel" \
+      --junit-path ${ARTIFACT_DIR}/junit_report.xml
+  from: src
+```
+
+## Makefile Commands
+
+| Target | Description |
+|--------|-------------|
+| `make build` | Builds the operator binary. |
+| `make tests-ext-build` | Builds the test extension binary. |
+| `make tests-ext-update` | Updates the metadata JSON file and cleans machine-specific codeLocations. |
+| `make verify` | Runs formatting, vet, and linter. |
+
+**Note:** Metadata is stored in: `.openshift-tests-extension/cluster-openshift-controller-manager-operator.json`
+
+## FAQ
+
+### Why don't we have a Dockerfile for `cluster-openshift-controller-manager-operator-tests-ext`?
+
+We do not provide a Dockerfile for `cluster-openshift-controller-manager-operator-tests-ext` because building and shipping a
+standalone image for this test binary would introduce unnecessary complexity.
+
+Technically, it is possible to create a new OpenShift component just for the
+tests and add a corresponding test image to the payload. However, doing so requires
+onboarding a new component, setting up build pipelines, and maintaining image promotion
+and test configuration — all of which adds overhead.
+
+From the OpenShift architecture point of view:
+
+1. Tests for payload components are part of the product. Many users (such as storage vendors, or third-party CNIs)
+rely on these tests to validate that their solutions are compatible and conformant with OpenShift.
+
+2. Adding new images to the payload comes with significant overhead and cost.
+It is generally preferred to include tests in the same image as the component
+being tested whenever possible.
+
+### Why do we need to run `make tests-ext-update`?
+
+Running `make tests-ext-update` ensures that each test gets a unique and stable **TestID** over time.
+
+The TestID is used to identify tests across the OpenShift CI/CD pipeline and reporting tools like Sippy.
+It helps track test results, detect regressions, and ensures the correct tests are
+executed and reported.
+
+This step is important whenever you add, rename, or delete a test.
+More information:
+- https://github.com/openshift/enhancements/blob/master/enhancements/testing/openshift-tests-extension.md#test-id
+- https://github.com/openshift-eng/ci-test-mapping
+
+### How to get help with OTE?
+
+For help with the OpenShift Tests Extension (OTE), you can reach out on the #wg-openshift-tests-extension Slack channel.
+
+## Test Implementation Details
+
+The OTE implementation uses the registry-based approach from the `openshift-tests-extension` framework:
+
+- **Registry Pattern**: Uses `extension.NewRegistry()` and `extension.NewExtension()` for clean, maintainable code
+- **JUnit Integration**: Generates both OTE framework and Ginkgo JUnit XML reports
+- **Test Discovery**: Automatically discovers and registers Ginkgo tests from the `test/extended` package
+- **CI Ready**: Includes proper JUnit reporter configuration for CI integration
+
+## Architecture
+
+The OTE binary (`cluster-openshift-controller-manager-operator-tests-ext`) provides:
+
+1. **Test Discovery**: Lists all available tests and suites
+2. **Test Execution**: Runs individual tests or entire suites
+3. **Result Reporting**: Generates JSON and JUnit XML output
+4. **CI Integration**: Provides standardized output formats for CI systems
+
+The implementation follows OpenShift best practices and integrates seamlessly with the existing CI/CD pipeline.