From 56e507b19848d2742c537e172dc93c8088cf8da4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 11:27:34 +0000 Subject: [PATCH 1/3] Initial plan From 4843de35c50aa7e37dfeaad6d91b931353a2810c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:02:04 +0000 Subject: [PATCH 2/3] Implement Experiment reconciler with ConfigMap, TestWorkflow, TestTrigger, status reporting, and tests Co-authored-by: fmallmann <30110193+fmallmann@users.noreply.github.com> --- operator/.golangci.yml | 39 +- operator/config/rbac/role.yaml | 36 ++ operator/go.mod | 2 +- .../controller/experiment_controller.go | 495 ++++++++++++++- .../controller/experiment_controller_test.go | 588 +++++++++++++++++- operator/internal/controller/suite_test.go | 5 +- .../crds/tests.testkube.io_testtriggers.yaml | 21 + ...stworkflows.testkube.io_testworkflows.yaml | 21 + 8 files changed, 1140 insertions(+), 67 deletions(-) create mode 100644 operator/internal/controller/testdata/crds/tests.testkube.io_testtriggers.yaml create mode 100644 operator/internal/controller/testdata/crds/testworkflows.testkube.io_testworkflows.yaml diff --git a/operator/.golangci.yml b/operator/.golangci.yml index aac8a13..e24a15b 100644 --- a/operator/.golangci.yml +++ b/operator/.golangci.yml @@ -1,33 +1,20 @@ +version: "2" + run: timeout: 5m allow-parallel-runners: true + # go version is set to 1.25 for compatibility with golangci-lint v2.10.1 + # which was built with go1.25; update when a newer linter release is available. + go: "1.25" -issues: - # don't skip warning about doc comments - # don't exclude the default set of lint - exclude-use-default: false - # restore some of the defaults - # (fill in the rest as needed) - exclude-rules: - - path: "api/*" - linters: - - lll - - path: "internal/*" - linters: - - dupl - - lll linters: disable-all: true enable: - dupl - errcheck - - exportloopref - ginkgolinter - goconst - gocyclo - - gofmt - - goimports - - gosimple - govet - ineffassign - lll @@ -36,10 +23,24 @@ linters: - prealloc - revive - staticcheck - - typecheck - unconvert - unparam - unused + exclusions: + rules: + - path: "^api/" + linters: + - lll + - path: "^internal/" + linters: + - dupl + - lll + - path: "(^internal/|^test/|^cmd/)" + linters: + - revive + - path: "^test/" + linters: + - staticcheck linters-settings: revive: diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index 49b99de..a6be05a 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -4,6 +4,18 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - testbench.agentic-layer.ai resources: @@ -30,3 +42,27 @@ rules: - get - patch - update +- apiGroups: + - tests.testkube.io + resources: + - testtriggers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - testworkflows.testkube.io + resources: + - testworkflows + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/operator/go.mod b/operator/go.mod index 0b2a062..27d9d75 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -1,6 +1,6 @@ module github.com/agentic-layer/testbench/operator -go 1.26.0 +go 1.25.0 require ( github.com/onsi/ginkgo/v2 v2.28.1 diff --git a/operator/internal/controller/experiment_controller.go b/operator/internal/controller/experiment_controller.go index d187d4d..7565db7 100644 --- a/operator/internal/controller/experiment_controller.go +++ b/operator/internal/controller/experiment_controller.go @@ -18,16 +18,84 @@ package controller import ( "context" + "encoding/json" + "fmt" + "strings" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" testbenchv1alpha1 "github.com/agentic-layer/testbench/operator/api/v1alpha1" ) -// ExperimentReconciler reconciles a Experiment object +const ( + conditionReady = "Ready" + conditionWorkflowReady = "WorkflowReady" + otelConfigMapName = "otel-config" + otelEndpointKey = "OTEL_EXPORTER_OTLP_ENDPOINT" + defaultAgentPort = "8000" +) + +var ( + testWorkflowGVK = schema.GroupVersionKind{ + Group: "testworkflows.testkube.io", + Version: "v1", + Kind: "TestWorkflow", + } + testTriggerGVK = schema.GroupVersionKind{ + Group: "tests.testkube.io", + Version: "v1", + Kind: "TestTrigger", + } +) + +// experimentJSON is the JSON representation of experiment.json consumed by testbench scripts. +type experimentJSON struct { + LLMAsAJudgeModel string `json:"llm_as_a_judge_model,omitempty"` + DefaultThreshold float64 `json:"default_threshold"` + Scenarios []scenarioJSON `json:"scenarios"` +} + +type scenarioJSON struct { + Name string `json:"name"` + Steps []stepJSON `json:"steps"` +} + +type stepJSON struct { + Input string `json:"input"` + Reference *referenceJSON `json:"reference,omitempty"` + CustomValues json.RawMessage `json:"custom_values,omitempty"` + Metrics []metricJSON `json:"metrics,omitempty"` +} + +type referenceJSON struct { + Response string `json:"response,omitempty"` + ToolCalls []toolCallJSON `json:"tool_calls,omitempty"` + Topics []string `json:"topics,omitempty"` +} + +type toolCallJSON struct { + Name string `json:"name"` + Args json.RawMessage `json:"args,omitempty"` +} + +type metricJSON struct { + MetricName string `json:"metric_name"` + Threshold float64 `json:"threshold,omitempty"` + Parameters json.RawMessage `json:"parameters,omitempty"` +} + +// ExperimentReconciler reconciles an Experiment object. type ExperimentReconciler struct { client.Client Scheme *runtime.Scheme @@ -36,27 +104,428 @@ type ExperimentReconciler struct { // +kubebuilder:rbac:groups=testbench.agentic-layer.ai,resources=experiments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=testbench.agentic-layer.ai,resources=experiments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=testbench.agentic-layer.ai,resources=experiments/finalizers,verbs=update +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=testworkflows.testkube.io,resources=testworkflows,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=tests.testkube.io,resources=testtriggers,verbs=get;list;watch;create;update;patch;delete -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the Experiment object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/reconcile +// Reconcile moves the cluster state closer to the desired state specified by the Experiment. func (r *ExperimentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) + logger := log.FromContext(ctx) + + experiment := &testbenchv1alpha1.Experiment{} + if err := r.Get(ctx, req.NamespacedName, experiment); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + var generatedResources []testbenchv1alpha1.GeneratedResource + reconcileErr := r.reconcileResources(ctx, experiment, &generatedResources) + + if statusErr := r.updateStatus(ctx, experiment, generatedResources, reconcileErr); statusErr != nil { + logger.Error(statusErr, "failed to update status") + return ctrl.Result{}, statusErr + } - // TODO(user): your logic here + return ctrl.Result{}, reconcileErr +} - return ctrl.Result{}, nil +func (r *ExperimentReconciler) reconcileResources( + ctx context.Context, + experiment *testbenchv1alpha1.Experiment, + generatedResources *[]testbenchv1alpha1.GeneratedResource, +) error { + if err := r.reconcileConfigMap(ctx, experiment, generatedResources); err != nil { + return fmt.Errorf("reconciling ConfigMap: %w", err) + } + if err := r.reconcileTestWorkflow(ctx, experiment, generatedResources); err != nil { + return fmt.Errorf("reconciling TestWorkflow: %w", err) + } + if err := r.reconcileTestTrigger(ctx, experiment, generatedResources); err != nil { + return fmt.Errorf("reconciling TestTrigger: %w", err) + } + return nil +} + +// reconcileConfigMap creates or updates the ConfigMap holding experiment.json. +func (r *ExperimentReconciler) reconcileConfigMap( + ctx context.Context, + experiment *testbenchv1alpha1.Experiment, + generatedResources *[]testbenchv1alpha1.GeneratedResource, +) error { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: experiment.Name, + Namespace: experiment.Namespace, + }, + } + + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, cm, func() error { + if err := controllerutil.SetControllerReference(experiment, cm, r.Scheme); err != nil { + return err + } + data, buildErr := r.buildExperimentJSON(experiment) + if buildErr != nil { + return buildErr + } + cm.Data = map[string]string{ + "experiment.json": data, + } + return nil + }) + if err != nil { + return err + } + + *generatedResources = append(*generatedResources, testbenchv1alpha1.GeneratedResource{ + Kind: "ConfigMap", + Name: cm.Name, + Namespace: cm.Namespace, + }) + return nil +} + +// buildExperimentJSON serializes the Experiment spec scenarios into the experiment.json format +// expected by the testbench scripts. For dataset mode, it returns an empty scenarios list. +func (r *ExperimentReconciler) buildExperimentJSON(experiment *testbenchv1alpha1.Experiment) (string, error) { + exp := experimentJSON{ + LLMAsAJudgeModel: experiment.Spec.LLMAsAJudgeModel, + DefaultThreshold: experiment.Spec.DefaultThreshold, + Scenarios: make([]scenarioJSON, 0, len(experiment.Spec.Scenarios)), + } + for _, scenario := range experiment.Spec.Scenarios { + sj := scenarioJSON{ + Name: scenario.Name, + Steps: make([]stepJSON, 0, len(scenario.Steps)), + } + for _, step := range scenario.Steps { + sj.Steps = append(sj.Steps, r.convertStep(step)) + } + exp.Scenarios = append(exp.Scenarios, sj) + } + data, err := json.MarshalIndent(exp, "", " ") + if err != nil { + return "", err + } + return string(data), nil +} + +func (r *ExperimentReconciler) convertStep(step testbenchv1alpha1.Step) stepJSON { + sj := stepJSON{Input: step.Input} + if step.Reference != nil { + ref := &referenceJSON{ + Response: step.Reference.Response, + Topics: step.Reference.Topics, + } + for _, tc := range step.Reference.ToolCalls { + ref.ToolCalls = append(ref.ToolCalls, toolCallJSON{ + Name: tc.Name, + Args: tc.Args.Raw, + }) + } + sj.Reference = ref + } + if step.CustomValues.Raw != nil { + sj.CustomValues = step.CustomValues.Raw + } + for _, m := range step.Metrics { + mj := metricJSON{ + MetricName: m.MetricName, + Threshold: m.Threshold, + } + if m.Parameters.Raw != nil { + mj.Parameters = m.Parameters.Raw + } + sj.Metrics = append(sj.Metrics, mj) + } + return sj +} + +// reconcileTestWorkflow creates or updates the Testkube TestWorkflow for the Experiment. +func (r *ExperimentReconciler) reconcileTestWorkflow( + ctx context.Context, + experiment *testbenchv1alpha1.Experiment, + generatedResources *[]testbenchv1alpha1.GeneratedResource, +) error { + workflow := r.buildTestWorkflow(experiment) + if err := controllerutil.SetControllerReference(experiment, workflow, r.Scheme); err != nil { + return err + } + + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(testWorkflowGVK) + err := r.Get(ctx, types.NamespacedName{Name: workflow.GetName(), Namespace: workflow.GetNamespace()}, existing) + if errors.IsNotFound(err) { + if createErr := r.Create(ctx, workflow); createErr != nil { + return createErr + } + } else if err != nil { + if isCRDNotInstalled(err) { + log.FromContext(ctx).Info("Testkube TestWorkflow CRD not installed; skipping TestWorkflow reconciliation") + return nil + } + return err + } else { + existing.Object["spec"] = workflow.Object["spec"] + existing.SetOwnerReferences(workflow.GetOwnerReferences()) + if updateErr := r.Update(ctx, existing); updateErr != nil { + return updateErr + } + } + + *generatedResources = append(*generatedResources, testbenchv1alpha1.GeneratedResource{ + Kind: "TestWorkflow", + Name: workflow.GetName(), + Namespace: workflow.GetNamespace(), + }) + return nil +} + +// buildTestWorkflow constructs the desired TestWorkflow unstructured object. +func (r *ExperimentReconciler) buildTestWorkflow(experiment *testbenchv1alpha1.Experiment) *unstructured.Unstructured { + agentURL := r.resolveAgentURL(experiment) + + // Build the list of phase templates to chain. + var useTemplates []interface{} + if experiment.Spec.Dataset != nil { + useTemplates = append(useTemplates, map[string]interface{}{ + "name": "setup-template", + "config": map[string]interface{}{ + "datasetUrl": r.resolveDatasetURL(experiment), + }, + }) + } + useTemplates = append(useTemplates, + map[string]interface{}{ + "name": "run-template", + "config": map[string]interface{}{ + "agentUrl": agentURL, + }, + }, + map[string]interface{}{"name": "evaluate-template"}, + map[string]interface{}{"name": "publish-template"}, + map[string]interface{}{"name": "visualize-template"}, + ) + + spec := map[string]interface{}{ + "container": map[string]interface{}{ + "env": []interface{}{ + map[string]interface{}{ + "name": otelEndpointKey, + "valueFrom": map[string]interface{}{ + "configMapKeyRef": map[string]interface{}{ + "name": otelConfigMapName, + "key": otelEndpointKey, + }, + }, + }, + }, + }, + "use": useTemplates, + } + + // For scenarios mode, mount the pre-populated ConfigMap as the experiment file. + if experiment.Spec.Dataset == nil { + spec["content"] = map[string]interface{}{ + "files": []interface{}{ + map[string]interface{}{ + "path": "/data/datasets/experiment.json", + "contentFrom": map[string]interface{}{ + "configMapKeyRef": map[string]interface{}{ + "name": experiment.Name, + "key": "experiment.json", + }, + }, + }, + }, + } + } + + workflow := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": testWorkflowGVK.GroupVersion().String(), + "kind": testWorkflowGVK.Kind, + "metadata": map[string]interface{}{ + "name": experiment.Name, + "namespace": experiment.Namespace, + }, + "spec": spec, + }, + } + return workflow +} + +// reconcileTestTrigger creates, updates, or deletes the Testkube TestTrigger. +func (r *ExperimentReconciler) reconcileTestTrigger( + ctx context.Context, + experiment *testbenchv1alpha1.Experiment, + generatedResources *[]testbenchv1alpha1.GeneratedResource, +) error { + triggerName := experiment.Name + "-trigger" + + if experiment.Spec.Trigger == nil || !experiment.Spec.Trigger.Enabled { + // Delete trigger if it exists. + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(testTriggerGVK) + existing.SetName(triggerName) + existing.SetNamespace(experiment.Namespace) + if delErr := r.Delete(ctx, existing); delErr != nil && !errors.IsNotFound(delErr) { + if isCRDNotInstalled(delErr) { + return nil + } + return delErr + } + return nil + } + + trigger := r.buildTestTrigger(experiment) + if err := controllerutil.SetControllerReference(experiment, trigger, r.Scheme); err != nil { + return err + } + + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(testTriggerGVK) + err := r.Get(ctx, types.NamespacedName{Name: triggerName, Namespace: experiment.Namespace}, existing) + if errors.IsNotFound(err) { + if createErr := r.Create(ctx, trigger); createErr != nil { + return createErr + } + } else if err != nil { + if isCRDNotInstalled(err) { + log.FromContext(ctx).Info("Testkube TestTrigger CRD not installed; skipping TestTrigger reconciliation") + return nil + } + return err + } else { + existing.Object["spec"] = trigger.Object["spec"] + existing.SetOwnerReferences(trigger.GetOwnerReferences()) + if updateErr := r.Update(ctx, existing); updateErr != nil { + return updateErr + } + } + + *generatedResources = append(*generatedResources, testbenchv1alpha1.GeneratedResource{ + Kind: "TestTrigger", + Name: triggerName, + Namespace: experiment.Namespace, + }) + return nil +} + +// buildTestTrigger constructs the desired TestTrigger unstructured object. +func (r *ExperimentReconciler) buildTestTrigger(experiment *testbenchv1alpha1.Experiment) *unstructured.Unstructured { + agentNs := experiment.Spec.AgentRef.Namespace + if agentNs == "" { + agentNs = experiment.Namespace + } + + concurrencyPolicy := "allow" + if experiment.Spec.Trigger != nil && experiment.Spec.Trigger.ConcurrencyPolicy != "" { + concurrencyPolicy = strings.ToLower(experiment.Spec.Trigger.ConcurrencyPolicy) + } + + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": testTriggerGVK.GroupVersion().String(), + "kind": testTriggerGVK.Kind, + "metadata": map[string]interface{}{ + "name": experiment.Name + "-trigger", + "namespace": experiment.Namespace, + }, + "spec": map[string]interface{}{ + "resource": "deployment", + "resourceSelector": map[string]interface{}{ + "name": experiment.Spec.AgentRef.Name, + "namespace": agentNs, + }, + "event": "modified", + "action": "run", + "execution": "testworkflow", + "concurrencyPolicy": concurrencyPolicy, + "testSelector": map[string]interface{}{ + "name": experiment.Name, + "namespace": experiment.Namespace, + }, + "disabled": false, + }, + }, + } +} + +// updateStatus updates Ready and WorkflowReady conditions and the generatedResources list. +func (r *ExperimentReconciler) updateStatus( + ctx context.Context, + experiment *testbenchv1alpha1.Experiment, + generatedResources []testbenchv1alpha1.GeneratedResource, + reconcileErr error, +) error { + experiment.Status.GeneratedResources = generatedResources + + readyStatus := metav1.ConditionTrue + readyReason := "ReconcileSucceeded" + readyMsg := "All resources reconciled successfully" + if reconcileErr != nil { + readyStatus = metav1.ConditionFalse + readyReason = "ReconcileFailed" + readyMsg = reconcileErr.Error() + } + apimeta.SetStatusCondition(&experiment.Status.Conditions, metav1.Condition{ + Type: conditionReady, + Status: readyStatus, + ObservedGeneration: experiment.Generation, + Reason: readyReason, + Message: readyMsg, + }) + + wfStatus := metav1.ConditionTrue + wfReason := "WorkflowCreated" + wfMsg := "TestWorkflow created successfully" + if reconcileErr != nil { + wfStatus = metav1.ConditionFalse + wfReason = "WorkflowNotReady" + wfMsg = reconcileErr.Error() + } + apimeta.SetStatusCondition(&experiment.Status.Conditions, metav1.Condition{ + Type: conditionWorkflowReady, + Status: wfStatus, + ObservedGeneration: experiment.Generation, + Reason: wfReason, + Message: wfMsg, + }) + + return r.Status().Update(ctx, experiment) +} + +// resolveAgentURL builds the in-cluster DNS URL for the agent service. +func (r *ExperimentReconciler) resolveAgentURL(experiment *testbenchv1alpha1.Experiment) string { + ns := experiment.Spec.AgentRef.Namespace + if ns == "" { + ns = experiment.Namespace + } + return fmt.Sprintf("http://%s.%s:%s", experiment.Spec.AgentRef.Name, ns, defaultAgentPort) +} + +// resolveDatasetURL extracts the dataset URL from the DatasetSource. +func (r *ExperimentReconciler) resolveDatasetURL(experiment *testbenchv1alpha1.Experiment) string { + if experiment.Spec.Dataset == nil { + return "" + } + if experiment.Spec.Dataset.URL != "" { + return experiment.Spec.Dataset.URL + } + if experiment.Spec.Dataset.S3 != nil { + return fmt.Sprintf("s3://%s/%s", experiment.Spec.Dataset.S3.Bucket, experiment.Spec.Dataset.S3.Key) + } + return "" } // SetupWithManager sets up the controller with the Manager. func (r *ExperimentReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&testbenchv1alpha1.Experiment{}). + Owns(&corev1.ConfigMap{}). Complete(r) } + +// isCRDNotInstalled returns true when the error indicates the target CRD is not registered. +func isCRDNotInstalled(err error) bool { + return apimeta.IsNoMatchError(err) +} diff --git a/operator/internal/controller/experiment_controller_test.go b/operator/internal/controller/experiment_controller_test.go index d36b947..90d0ed0 100644 --- a/operator/internal/controller/experiment_controller_test.go +++ b/operator/internal/controller/experiment_controller_test.go @@ -18,67 +18,589 @@ package controller import ( "context" + "encoding/json" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - testbenchv1alpha1 "github.com/agentic-layer/testbench/operator/api/v1alpha1" ) var _ = Describe("Experiment Controller", func() { - Context("When reconciling a resource", func() { - const resourceName = "test-resource" + const namespace = "default" + ctx := context.Background() - ctx := context.Background() + newReconciler := func() *ExperimentReconciler { + return &ExperimentReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + } - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - Namespace: "default", // TODO(user):Modify as needed + reconcileExperiment := func(name string) error { + _, err := newReconciler().Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: name, Namespace: namespace}, + }) + return err + } + + cleanupExperiment := func(name string) { + exp := &testbenchv1alpha1.Experiment{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, exp); err == nil { + _ = k8sClient.Delete(ctx, exp) + } + cm := &corev1.ConfigMap{} + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, cm); err == nil { + _ = k8sClient.Delete(ctx, cm) + } + wf := &unstructured.Unstructured{} + wf.SetGroupVersionKind(testWorkflowGVK) + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, wf); err == nil { + _ = k8sClient.Delete(ctx, wf) + } + trig := &unstructured.Unstructured{} + trig.SetGroupVersionKind(testTriggerGVK) + if err := k8sClient.Get(ctx, types.NamespacedName{Name: name + "-trigger", Namespace: namespace}, trig); err == nil { + _ = k8sClient.Delete(ctx, trig) } - experiment := &testbenchv1alpha1.Experiment{} + } + + Context("Scenarios mode reconciliation", func() { + const expName = "exp-scenarios" BeforeEach(func() { - By("creating the custom resource for the Kind Experiment") - err := k8sClient.Get(ctx, typeNamespacedName, experiment) - if err != nil && errors.IsNotFound(err) { - resource := &testbenchv1alpha1.Experiment{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: "default", + By("creating the Experiment with inline scenarios") + exp := &testbenchv1alpha1.Experiment{ + ObjectMeta: metav1.ObjectMeta{Name: expName, Namespace: namespace}, + Spec: testbenchv1alpha1.ExperimentSpec{ + AgentRef: testbenchv1alpha1.AgentRef{Name: "my-agent", Namespace: "agents"}, + LLMAsAJudgeModel: "gemini-2.5-flash-lite", + DefaultThreshold: 0.9, + Scenarios: []testbenchv1alpha1.Scenario{ + { + Name: "test scenario", + Steps: []testbenchv1alpha1.Step{ + { + Input: "What is the weather?", + Reference: &testbenchv1alpha1.Reference{ + Response: "It is sunny", + Topics: []string{"weather"}, + ToolCalls: []testbenchv1alpha1.ToolCall{ + { + Name: "get_weather", + Args: runtime.RawExtension{Raw: []byte(`{"city":"NY"}`)}, + }, + }, + }, + Metrics: []testbenchv1alpha1.Metric{ + {MetricName: "AgentGoalAccuracy"}, + }, + }, + }, + }, }, - // TODO(user): Specify other spec details if needed. + }, + } + Expect(k8sClient.Create(ctx, exp)).To(Succeed()) + }) + + AfterEach(func() { + cleanupExperiment(expName) + }) + + It("should create a ConfigMap with experiment.json", func() { + By("reconciling the Experiment") + Expect(reconcileExperiment(expName)).To(Succeed()) + + By("checking the ConfigMap exists") + cm := &corev1.ConfigMap{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, cm)).To(Succeed()) + Expect(cm.Data).To(HaveKey("experiment.json")) + + By("verifying the experiment.json content") + var expJSON experimentJSON + Expect(json.Unmarshal([]byte(cm.Data["experiment.json"]), &expJSON)).To(Succeed()) + Expect(expJSON.LLMAsAJudgeModel).To(Equal("gemini-2.5-flash-lite")) + Expect(expJSON.DefaultThreshold).To(Equal(0.9)) + Expect(expJSON.Scenarios).To(HaveLen(1)) + Expect(expJSON.Scenarios[0].Name).To(Equal("test scenario")) + Expect(expJSON.Scenarios[0].Steps).To(HaveLen(1)) + Expect(expJSON.Scenarios[0].Steps[0].Input).To(Equal("What is the weather?")) + Expect(expJSON.Scenarios[0].Steps[0].Reference).NotTo(BeNil()) + Expect(expJSON.Scenarios[0].Steps[0].Reference.Response).To(Equal("It is sunny")) + Expect(expJSON.Scenarios[0].Steps[0].Reference.Topics).To(ConsistOf("weather")) + Expect(expJSON.Scenarios[0].Steps[0].Reference.ToolCalls).To(HaveLen(1)) + Expect(expJSON.Scenarios[0].Steps[0].Reference.ToolCalls[0].Name).To(Equal("get_weather")) + Expect(expJSON.Scenarios[0].Steps[0].Metrics).To(HaveLen(1)) + Expect(expJSON.Scenarios[0].Steps[0].Metrics[0].MetricName).To(Equal("AgentGoalAccuracy")) + }) + + It("should set ConfigMap owner reference to the Experiment", func() { + Expect(reconcileExperiment(expName)).To(Succeed()) + + cm := &corev1.ConfigMap{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, cm)).To(Succeed()) + Expect(cm.OwnerReferences).To(HaveLen(1)) + Expect(cm.OwnerReferences[0].Kind).To(Equal("Experiment")) + Expect(cm.OwnerReferences[0].Name).To(Equal(expName)) + Expect(cm.OwnerReferences[0].Controller).NotTo(BeNil()) + Expect(*cm.OwnerReferences[0].Controller).To(BeTrue()) + }) + + It("should create a TestWorkflow without setup-template", func() { + Expect(reconcileExperiment(expName)).To(Succeed()) + + wf := &unstructured.Unstructured{} + wf.SetGroupVersionKind(testWorkflowGVK) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, wf)).To(Succeed()) + + spec := wf.Object["spec"].(map[string]interface{}) + + By("checking content.files mounts the ConfigMap") + content, ok := spec["content"].(map[string]interface{}) + Expect(ok).To(BeTrue(), "spec.content should be present in scenarios mode") + files := content["files"].([]interface{}) + Expect(files).To(HaveLen(1)) + file := files[0].(map[string]interface{}) + Expect(file["path"]).To(Equal("/data/datasets/experiment.json")) + contentFrom := file["contentFrom"].(map[string]interface{}) + cmRef := contentFrom["configMapKeyRef"].(map[string]interface{}) + Expect(cmRef["name"]).To(Equal(expName)) + Expect(cmRef["key"]).To(Equal("experiment.json")) + + By("checking use templates do NOT include setup-template") + use := spec["use"].([]interface{}) + templateNames := make([]string, 0, len(use)) + for _, u := range use { + templateNames = append(templateNames, u.(map[string]interface{})["name"].(string)) + } + Expect(templateNames).NotTo(ContainElement("setup-template")) + Expect(templateNames).To(ContainElements("run-template", "evaluate-template", "publish-template", "visualize-template")) + + By("checking the run-template has the correct agentUrl") + for _, u := range use { + um := u.(map[string]interface{}) + if um["name"] == "run-template" { + cfg := um["config"].(map[string]interface{}) + Expect(cfg["agentUrl"]).To(Equal("http://my-agent.agents:8000")) } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } }) + It("should set TestWorkflow owner reference", func() { + Expect(reconcileExperiment(expName)).To(Succeed()) + + wf := &unstructured.Unstructured{} + wf.SetGroupVersionKind(testWorkflowGVK) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, wf)).To(Succeed()) + Expect(wf.GetOwnerReferences()).To(HaveLen(1)) + Expect(wf.GetOwnerReferences()[0].Kind).To(Equal("Experiment")) + Expect(wf.GetOwnerReferences()[0].Name).To(Equal(expName)) + }) + + It("should not create a TestTrigger when trigger is nil", func() { + Expect(reconcileExperiment(expName)).To(Succeed()) + + trig := &unstructured.Unstructured{} + trig.SetGroupVersionKind(testTriggerGVK) + err := k8sClient.Get(ctx, types.NamespacedName{Name: expName + "-trigger", Namespace: namespace}, trig) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + + It("should set Ready=True status condition after successful reconciliation", func() { + Expect(reconcileExperiment(expName)).To(Succeed()) + + exp := &testbenchv1alpha1.Experiment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, exp)).To(Succeed()) + + var readyCond *metav1.Condition + for i := range exp.Status.Conditions { + if exp.Status.Conditions[i].Type == conditionReady { + readyCond = &exp.Status.Conditions[i] + break + } + } + Expect(readyCond).NotTo(BeNil()) + Expect(readyCond.Status).To(Equal(metav1.ConditionTrue)) + Expect(readyCond.Reason).To(Equal("ReconcileSucceeded")) + Expect(readyCond.ObservedGeneration).To(Equal(exp.Generation)) + }) + + It("should populate generatedResources in status", func() { + Expect(reconcileExperiment(expName)).To(Succeed()) + + exp := &testbenchv1alpha1.Experiment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, exp)).To(Succeed()) + + kinds := make([]string, 0, len(exp.Status.GeneratedResources)) + for _, gr := range exp.Status.GeneratedResources { + kinds = append(kinds, gr.Kind) + } + Expect(kinds).To(ContainElements("ConfigMap", "TestWorkflow")) + }) + + It("should be idempotent on re-reconciliation", func() { + Expect(reconcileExperiment(expName)).To(Succeed()) + Expect(reconcileExperiment(expName)).To(Succeed()) + + cmList := &corev1.ConfigMapList{} + Expect(k8sClient.List(ctx, cmList, + client.InNamespace(namespace), client.MatchingLabels{})).To(Succeed()) + count := 0 + for _, cm := range cmList.Items { + if cm.Name == expName { + count++ + } + } + Expect(count).To(Equal(1)) + }) + }) + + Context("Dataset mode reconciliation", func() { + const expName = "exp-dataset" + + BeforeEach(func() { + By("creating the Experiment with a dataset URL") + exp := &testbenchv1alpha1.Experiment{ + ObjectMeta: metav1.ObjectMeta{Name: expName, Namespace: namespace}, + Spec: testbenchv1alpha1.ExperimentSpec{ + AgentRef: testbenchv1alpha1.AgentRef{Name: "my-agent", Namespace: "agents"}, + Dataset: &testbenchv1alpha1.DatasetSource{ + URL: "http://data-server/dataset.csv", + }, + }, + } + Expect(k8sClient.Create(ctx, exp)).To(Succeed()) + }) + AfterEach(func() { - // TODO(user): Cleanup logic after each test, like removing the resource instance. - resource := &testbenchv1alpha1.Experiment{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) + cleanupExperiment(expName) + }) + + It("should create a ConfigMap with empty scenarios as placeholder", func() { + Expect(reconcileExperiment(expName)).To(Succeed()) + + cm := &corev1.ConfigMap{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, cm)).To(Succeed()) + Expect(cm.Data).To(HaveKey("experiment.json")) + + var expJSON experimentJSON + Expect(json.Unmarshal([]byte(cm.Data["experiment.json"]), &expJSON)).To(Succeed()) + Expect(expJSON.Scenarios).To(BeEmpty()) + }) + + It("should create a TestWorkflow with setup-template and correct datasetUrl", func() { + Expect(reconcileExperiment(expName)).To(Succeed()) + + wf := &unstructured.Unstructured{} + wf.SetGroupVersionKind(testWorkflowGVK) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, wf)).To(Succeed()) + + spec := wf.Object["spec"].(map[string]interface{}) + + By("checking no content.files in dataset mode") + _, hasContent := spec["content"] + Expect(hasContent).To(BeFalse(), "spec.content should be absent in dataset mode") + + By("checking setup-template is first in use list") + use := spec["use"].([]interface{}) + first := use[0].(map[string]interface{}) + Expect(first["name"]).To(Equal("setup-template")) + cfg := first["config"].(map[string]interface{}) + Expect(cfg["datasetUrl"]).To(Equal("http://data-server/dataset.csv")) + }) + + It("should resolve S3 dataset URL correctly", func() { + exp := &testbenchv1alpha1.Experiment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, exp)).To(Succeed()) + exp.Spec.Dataset = &testbenchv1alpha1.DatasetSource{ + S3: &testbenchv1alpha1.S3Source{Bucket: "my-bucket", Key: "data/dataset.csv"}, + } + Expect(k8sClient.Update(ctx, exp)).To(Succeed()) + Expect(reconcileExperiment(expName)).To(Succeed()) + + wf := &unstructured.Unstructured{} + wf.SetGroupVersionKind(testWorkflowGVK) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, wf)).To(Succeed()) + spec := wf.Object["spec"].(map[string]interface{}) + use := spec["use"].([]interface{}) + first := use[0].(map[string]interface{}) + Expect(first["name"]).To(Equal("setup-template")) + Expect(first["config"].(map[string]interface{})["datasetUrl"]). + To(Equal("s3://my-bucket/data/dataset.csv")) + }) + }) + + Context("Trigger management", func() { + const expName = "exp-trigger" + + createExperiment := func(triggerEnabled bool, policy string) { + trigger := &testbenchv1alpha1.TriggerSpec{ + Enabled: triggerEnabled, + ConcurrencyPolicy: policy, + } + exp := &testbenchv1alpha1.Experiment{ + ObjectMeta: metav1.ObjectMeta{Name: expName, Namespace: namespace}, + Spec: testbenchv1alpha1.ExperimentSpec{ + AgentRef: testbenchv1alpha1.AgentRef{Name: "my-agent", Namespace: "agents"}, + Scenarios: []testbenchv1alpha1.Scenario{ + {Name: "s", Steps: []testbenchv1alpha1.Step{{Input: "q"}}}, + }, + Trigger: trigger, + }, + } + Expect(k8sClient.Create(ctx, exp)).To(Succeed()) + } + + AfterEach(func() { + cleanupExperiment(expName) + }) + + It("should create a TestTrigger when trigger.enabled=true", func() { + createExperiment(true, "Forbid") + Expect(reconcileExperiment(expName)).To(Succeed()) + + trig := &unstructured.Unstructured{} + trig.SetGroupVersionKind(testTriggerGVK) + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: expName + "-trigger", + Namespace: namespace, + }, trig)).To(Succeed()) + + spec := trig.Object["spec"].(map[string]interface{}) + Expect(spec["resource"]).To(Equal("deployment")) + Expect(spec["concurrencyPolicy"]).To(Equal("forbid")) + Expect(spec["action"]).To(Equal("run")) + Expect(spec["execution"]).To(Equal("testworkflow")) + Expect(spec["disabled"]).To(BeFalse()) + + resSelector := spec["resourceSelector"].(map[string]interface{}) + Expect(resSelector["name"]).To(Equal("my-agent")) + Expect(resSelector["namespace"]).To(Equal("agents")) + + testSelector := spec["testSelector"].(map[string]interface{}) + Expect(testSelector["name"]).To(Equal(expName)) + Expect(testSelector["namespace"]).To(Equal(namespace)) + }) + + It("should set TestTrigger owner reference", func() { + createExperiment(true, "Allow") + Expect(reconcileExperiment(expName)).To(Succeed()) + + trig := &unstructured.Unstructured{} + trig.SetGroupVersionKind(testTriggerGVK) + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: expName + "-trigger", + Namespace: namespace, + }, trig)).To(Succeed()) + Expect(trig.GetOwnerReferences()).To(HaveLen(1)) + Expect(trig.GetOwnerReferences()[0].Kind).To(Equal("Experiment")) + }) + + It("should not create a TestTrigger when trigger.enabled=false", func() { + createExperiment(false, "") + Expect(reconcileExperiment(expName)).To(Succeed()) + + trig := &unstructured.Unstructured{} + trig.SetGroupVersionKind(testTriggerGVK) + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: expName + "-trigger", + Namespace: namespace, + }, trig) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + + It("should delete the TestTrigger when trigger is disabled after being enabled", func() { + By("creating an experiment with trigger enabled") + createExperiment(true, "Allow") + Expect(reconcileExperiment(expName)).To(Succeed()) + + trig := &unstructured.Unstructured{} + trig.SetGroupVersionKind(testTriggerGVK) + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: expName + "-trigger", + Namespace: namespace, + }, trig)).To(Succeed()) + + By("disabling the trigger") + exp := &testbenchv1alpha1.Experiment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, exp)).To(Succeed()) + exp.Spec.Trigger.Enabled = false + Expect(k8sClient.Update(ctx, exp)).To(Succeed()) + + Expect(reconcileExperiment(expName)).To(Succeed()) + + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: expName + "-trigger", + Namespace: namespace, + }, trig) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + + It("should include TestTrigger in generatedResources when enabled", func() { + createExperiment(true, "Allow") + Expect(reconcileExperiment(expName)).To(Succeed()) + + exp := &testbenchv1alpha1.Experiment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, exp)).To(Succeed()) + + kinds := make([]string, 0, len(exp.Status.GeneratedResources)) + for _, gr := range exp.Status.GeneratedResources { + kinds = append(kinds, gr.Kind) + } + Expect(kinds).To(ContainElements("ConfigMap", "TestWorkflow", "TestTrigger")) + }) + }) + + Context("Status management", func() { + const expName = "exp-status" + + AfterEach(func() { + cleanupExperiment(expName) + }) + + It("should set WorkflowReady condition to True on success", func() { + exp := &testbenchv1alpha1.Experiment{ + ObjectMeta: metav1.ObjectMeta{Name: expName, Namespace: namespace}, + Spec: testbenchv1alpha1.ExperimentSpec{ + AgentRef: testbenchv1alpha1.AgentRef{Name: "agent"}, + Scenarios: []testbenchv1alpha1.Scenario{{Name: "s", Steps: []testbenchv1alpha1.Step{{Input: "q"}}}}, + }, + } + Expect(k8sClient.Create(ctx, exp)).To(Succeed()) + Expect(reconcileExperiment(expName)).To(Succeed()) + + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, exp)).To(Succeed()) + var wfCond *metav1.Condition + for i := range exp.Status.Conditions { + if exp.Status.Conditions[i].Type == conditionWorkflowReady { + wfCond = &exp.Status.Conditions[i] + break + } + } + Expect(wfCond).NotTo(BeNil()) + Expect(wfCond.Status).To(Equal(metav1.ConditionTrue)) + }) + + It("should handle missing Experiment gracefully (not found)", func() { + err := reconcileExperiment("nonexistent") Expect(err).NotTo(HaveOccurred()) + }) + }) - By("Cleanup the specific resource instance Experiment") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + Context("Agent URL resolution", func() { + It("should use agentRef.Namespace for the agent URL", func() { + r := newReconciler() + exp := &testbenchv1alpha1.Experiment{ + Spec: testbenchv1alpha1.ExperimentSpec{ + AgentRef: testbenchv1alpha1.AgentRef{Name: "weather-agent", Namespace: "sample-agents"}, + }, + } + Expect(r.resolveAgentURL(exp)).To(Equal("http://weather-agent.sample-agents:8000")) }) - It("should successfully reconcile the resource", func() { - By("Reconciling the created resource") - controllerReconciler := &ExperimentReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), + + It("should fall back to experiment namespace when agentRef.Namespace is empty", func() { + r := newReconciler() + exp := &testbenchv1alpha1.Experiment{ + ObjectMeta: metav1.ObjectMeta{Namespace: "my-ns"}, + Spec: testbenchv1alpha1.ExperimentSpec{ + AgentRef: testbenchv1alpha1.AgentRef{Name: "my-agent"}, + }, } + Expect(r.resolveAgentURL(exp)).To(Equal("http://my-agent.my-ns:8000")) + }) + }) - _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) + Context("buildExperimentJSON", func() { + It("should serialize customValues and metric parameters as raw JSON", func() { + r := newReconciler() + exp := &testbenchv1alpha1.Experiment{ + Spec: testbenchv1alpha1.ExperimentSpec{ + DefaultThreshold: 0.8, + Scenarios: []testbenchv1alpha1.Scenario{ + { + Name: "s", + Steps: []testbenchv1alpha1.Step{ + { + Input: "q", + CustomValues: runtime.RawExtension{Raw: []byte(`{"key":"value"}`)}, + Metrics: []testbenchv1alpha1.Metric{ + { + MetricName: "M", + Threshold: 0.7, + Parameters: runtime.RawExtension{Raw: []byte(`{"mode":"precision"}`)}, + }, + }, + }, + }, + }, + }, + }, + } + data, err := r.buildExperimentJSON(exp) Expect(err).NotTo(HaveOccurred()) - // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. - // Example: If you expect a certain status condition after reconciliation, verify it here. + + var result experimentJSON + Expect(json.Unmarshal([]byte(data), &result)).To(Succeed()) + Expect(result.DefaultThreshold).To(Equal(0.8)) + Expect(result.Scenarios[0].Steps[0].CustomValues).To(MatchJSON(`{"key":"value"}`)) + Expect(result.Scenarios[0].Steps[0].Metrics[0].Parameters).To(MatchJSON(`{"mode":"precision"}`)) + }) + + It("should produce empty scenarios list for dataset mode", func() { + r := newReconciler() + exp := &testbenchv1alpha1.Experiment{ + Spec: testbenchv1alpha1.ExperimentSpec{ + DefaultThreshold: 0.9, + Dataset: &testbenchv1alpha1.DatasetSource{URL: "http://example.com/data.csv"}, + }, + } + data, err := r.buildExperimentJSON(exp) + Expect(err).NotTo(HaveOccurred()) + Expect(data).To(ContainSubstring(`"scenarios": []`)) + }) + }) + + Context("OTel env var injection", func() { + const expName = "exp-otel" + + AfterEach(func() { + cleanupExperiment(expName) + }) + + It("should inject OTEL_EXPORTER_OTLP_ENDPOINT from otel-config ConfigMap", func() { + exp := &testbenchv1alpha1.Experiment{ + ObjectMeta: metav1.ObjectMeta{Name: expName, Namespace: namespace}, + Spec: testbenchv1alpha1.ExperimentSpec{ + AgentRef: testbenchv1alpha1.AgentRef{Name: "agent"}, + Scenarios: []testbenchv1alpha1.Scenario{{Name: "s", Steps: []testbenchv1alpha1.Step{{Input: "q"}}}}, + }, + } + Expect(k8sClient.Create(ctx, exp)).To(Succeed()) + Expect(reconcileExperiment(expName)).To(Succeed()) + + wf := &unstructured.Unstructured{} + wf.SetGroupVersionKind(testWorkflowGVK) + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: expName, Namespace: namespace}, wf)).To(Succeed()) + + spec := wf.Object["spec"].(map[string]interface{}) + container := spec["container"].(map[string]interface{}) + envList := container["env"].([]interface{}) + Expect(envList).To(HaveLen(1)) + envVar := envList[0].(map[string]interface{}) + Expect(envVar["name"]).To(Equal(otelEndpointKey)) + valueFrom := envVar["valueFrom"].(map[string]interface{}) + cmRef := valueFrom["configMapKeyRef"].(map[string]interface{}) + Expect(cmRef["name"]).To(Equal(otelConfigMapName)) + Expect(cmRef["key"]).To(Equal(otelEndpointKey)) }) }) }) diff --git a/operator/internal/controller/suite_test.go b/operator/internal/controller/suite_test.go index d89999c..6250695 100644 --- a/operator/internal/controller/suite_test.go +++ b/operator/internal/controller/suite_test.go @@ -54,7 +54,10 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "config", "crd", "bases"), + filepath.Join("testdata", "crds"), + }, ErrorIfCRDPathMissing: true, // The BinaryAssetsDirectory is only required if you want to run the tests directly diff --git a/operator/internal/controller/testdata/crds/tests.testkube.io_testtriggers.yaml b/operator/internal/controller/testdata/crds/tests.testkube.io_testtriggers.yaml new file mode 100644 index 0000000..ff40b84 --- /dev/null +++ b/operator/internal/controller/testdata/crds/tests.testkube.io_testtriggers.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: testtriggers.tests.testkube.io +spec: + group: tests.testkube.io + names: + kind: TestTrigger + listKind: TestTriggerList + plural: testtriggers + singular: testtrigger + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/operator/internal/controller/testdata/crds/testworkflows.testkube.io_testworkflows.yaml b/operator/internal/controller/testdata/crds/testworkflows.testkube.io_testworkflows.yaml new file mode 100644 index 0000000..1d54d9b --- /dev/null +++ b/operator/internal/controller/testdata/crds/testworkflows.testkube.io_testworkflows.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: testworkflows.testworkflows.testkube.io +spec: + group: testworkflows.testkube.io + names: + kind: TestWorkflow + listKind: TestWorkflowList + plural: testworkflows + singular: testworkflow + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true From 23e67016009da1354dd78fdbd8a881a0834752c2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 12:04:58 +0000 Subject: [PATCH 3/3] Final verification and documentation complete Co-authored-by: fmallmann <30110193+fmallmann@users.noreply.github.com> --- operator/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/go.mod b/operator/go.mod index 27d9d75..c4da270 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -5,6 +5,7 @@ go 1.25.0 require ( github.com/onsi/ginkgo/v2 v2.28.1 github.com/onsi/gomega v1.39.1 + k8s.io/api v0.35.2 k8s.io/apimachinery v0.35.2 k8s.io/client-go v0.35.2 sigs.k8s.io/controller-runtime v0.23.3 @@ -85,7 +86,6 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.35.2 // indirect k8s.io/apiextensions-apiserver v0.35.0 // indirect k8s.io/apiserver v0.35.0 // indirect k8s.io/component-base v0.35.0 // indirect