(fleet) Add experiment rollout to fleet daemon

api/datadoghq/v2alpha1/datadogagent_types.go

@@ -2349,6 +2349,34 @@ type RemoteConfigConfiguration struct {
 	Features *DatadogFeatures `json:"features,omitempty"`
 }
 
+// ExperimentPhase represents the current phase of a Fleet Automation experiment.
+// +kubebuilder:validation:Enum=running;stopped;rollback;timeout;promoted;aborted
+type ExperimentPhase string
+
+const (
+	ExperimentPhaseRunning  ExperimentPhase = "running"
+	ExperimentPhaseStopped  ExperimentPhase = "stopped"
+	ExperimentPhaseRollback ExperimentPhase = "rollback"
+	ExperimentPhasePromoted ExperimentPhase = "promoted"
+	ExperimentPhaseAborted  ExperimentPhase = "aborted"
+	ExperimentPhaseTimeout  ExperimentPhase = "timeout"
+)
+
+// ExperimentStatus tracks the state of a Fleet Automation experiment.
+// +k8s:openapi-gen=true
+type ExperimentStatus struct {
+	// Phase is the current state of the experiment.
+	// +optional
+	Phase ExperimentPhase `json:"phase,omitempty"`
+	// ID is the unique experiment ID sent by Fleet Automation.
+	// +optional
+	ID string `json:"id,omitempty"`
+	// Generation is the DDA metadata.generation recorded when the experiment started.
+	// Used to detect manual spec changes during an experiment.
+	// +optional
+	Generation int64 `json:"generation,omitempty"`
+}
+
 // DatadogAgentStatus defines the observed state of DatadogAgent.
 // +k8s:openapi-gen=true
 type DatadogAgentStatus struct {
@@ -2376,6 +2404,9 @@ type DatadogAgentStatus struct {
 	// RemoteConfigConfiguration stores the configuration received from RemoteConfig.
 	// +optional
 	RemoteConfigConfiguration *RemoteConfigConfiguration `json:"remoteConfigConfiguration,omitempty"`
+	// Experiment tracks the state of an active or recent Fleet Automation experiment.
+	// +optional
+	Experiment *ExperimentStatus `json:"experiment,omitempty"`
 }
 
 // DatadogAgent defines Agent configuration, see reference https://github.com/DataDog/datadog-operator/blob/main/docs/configuration.v2alpha1.md

cmd/fleet-test/main.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	"github.com/DataDog/datadog-operator/api/datadoghq/v2alpha1"
+	"github.com/DataDog/datadog-operator/pkg/fleet"
+)
+
+func main() {
+	var (
+		action       string
+		experimentID string
+		namespace    string
+		name         string
+		patch        string
+	)
+
+	flag.StringVar(&action, "action", "", "start, stop, or promote")
+	flag.StringVar(&experimentID, "id", "", "experiment ID")
+	flag.StringVar(&namespace, "namespace", "datadog", "DDA namespace")
+	flag.StringVar(&name, "name", "datadog", "DDA name")
+	flag.StringVar(&patch, "patch", "", "JSON merge patch for start")
+	flag.Parse()
+
+	if action == "" {
+		fmt.Fprintln(os.Stderr, "Usage: fleet-test -action start|stop|promote -id <experiment-id> [-patch <json>]")
+		os.Exit(1)
+	}
+
+	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+	logger := ctrl.Log.WithName("fleet-test")
+
+	// Build K8s client
+	scheme := runtime.NewScheme()
+	_ = v2alpha1.AddToScheme(scheme)
+
+	cfg, err := ctrl.GetConfig()
+	if err != nil {
+		logger.Error(err, "Failed to get kubeconfig")
+		os.Exit(1)
+	}
+
+	kubeClient, err := client.New(cfg, client.Options{Scheme: scheme})
+	if err != nil {
+		logger.Error(err, "Failed to create K8s client")
+		os.Exit(1)
+	}
+
+	// Build the daemon (no RC client needed for direct testing)
+	daemon := fleet.NewDaemonForTesting(logger, kubeClient)
+
+	ctx := context.Background()
+
+	switch action {
+	case "start":
+		if patch == "" {
+			fmt.Fprintln(os.Stderr, "Error: -patch is required for start")
+			os.Exit(1)
+		}
+		if experimentID == "" {
+			fmt.Fprintln(os.Stderr, "Error: -id is required")
+			os.Exit(1)
+		}
+
+		// Build installer config with the patch
+		configID := "test-config-" + experimentID
+		daemon.InjectConfig(configID, namespace, name, json.RawMessage(patch))
+
+		fmt.Printf("Starting experiment %s on %s/%s\n", experimentID, namespace, name)
+		err = daemon.StartExperiment(ctx, experimentID, configID)
+
+	case "stop":
+		if experimentID == "" {
+			fmt.Fprintln(os.Stderr, "Error: -id is required")
+			os.Exit(1)
+		}
+		fmt.Printf("Stopping experiment %s\n", experimentID)
+		err = daemon.StopExperiment(ctx, experimentID)
+
+	case "promote":
+		if experimentID == "" {
+			fmt.Fprintln(os.Stderr, "Error: -id is required")
+			os.Exit(1)
+		}
+		fmt.Printf("Promoting experiment %s\n", experimentID)
+		err = daemon.PromoteExperiment(ctx, experimentID)
+
+	case "status":
+		dda := &v2alpha1.DatadogAgent{}
+		if getErr := kubeClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, dda); getErr != nil {
+			logger.Error(getErr, "Failed to get DDA")
+			os.Exit(1)
+		}
+		if dda.Status.Experiment == nil {
+			fmt.Println("No experiment running")
+		} else {
+			data, _ := json.MarshalIndent(dda.Status.Experiment, "", "  ")
+			fmt.Printf("Experiment:\n%s\n", string(data))
+		}
+		fmt.Printf("APM enabled: %v\n", dda.Spec.Features != nil && dda.Spec.Features.APM != nil && dda.Spec.Features.APM.Enabled != nil && *dda.Spec.Features.APM.Enabled)
+		return
+
+	default:
+		fmt.Fprintf(os.Stderr, "Unknown action: %s\n", action)
+		os.Exit(1)
+	}
+
+	if err != nil {
+		logger.Error(err, "Failed", "action", action)
+		os.Exit(1)
+	}
+	fmt.Printf("Success: %s experiment %s\n", action, experimentID)
+
+	// Print current state
+	dda := &v2alpha1.DatadogAgent{}
+	if getErr := kubeClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, dda); getErr == nil {
+		if dda.Status.Experiment != nil {
+			data, _ := json.MarshalIndent(dda.Status.Experiment, "", "  ")
+			fmt.Printf("Experiment status:\n%s\n", string(data))
+		}
+	}
+}
+
+// Unused but needed for the fleetManagementOperation import
+var _ = schema.GroupVersionKind{}

cmd/main.go

@@ -666,6 +666,6 @@ func setupAndStartHelmMetadataForwarder(logger logr.Logger, mgr manager.Manager,
 }
 
 func setupFleetDaemon(logger logr.Logger, mgr manager.Manager, rcClient remoteconfig.RCClient) error {
-	daemon := fleet.NewDaemon(logger.WithName("fleet"), rcClient)
+	daemon := fleet.NewDaemon(logger.WithName("fleet"), rcClient, mgr.GetClient())
 	return mgr.Add(daemon)
 }

config/crd/bases/v1/datadoghq.com_datadogagents.yaml

@@ -8474,6 +8474,29 @@ spec:
                   x-kubernetes-list-map-keys:
                     - type
                   x-kubernetes-list-type: map
+                experiment:
+                  description: Experiment tracks the state of an active or recent Fleet Automation experiment.
+                  properties:
+                    generation:
+                      description: |-
+                        Generation is the DDA metadata.generation recorded when the experiment started.
+                        Used to detect manual spec changes during an experiment.
+                      format: int64
+                      type: integer
+                    id:
+                      description: ID is the unique experiment ID sent by Fleet Automation.
+                      type: string
+                    phase:
+                      description: Phase is the current state of the experiment.
+                      enum:
+                        - running
+                        - stopped
+                        - rollback
+                        - timeout
+                        - promoted
+                        - aborted
+                      type: string
+                  type: object
                 otelAgentGateway:
                   description: The actual state of the OTel Agent Gateway as a deployment.
                   properties:

config/crd/bases/v1/datadoghq.com_datadogagents_v2alpha1.json

@@ -8175,6 +8175,34 @@
           ],
           "x-kubernetes-list-type": "map"
         },
+        "experiment": {
+          "additionalProperties": false,
+          "description": "Experiment tracks the state of an active or recent Fleet Automation experiment.",
+          "properties": {
+            "generation": {
+              "description": "Generation is the DDA metadata.generation recorded when the experiment started.\nUsed to detect manual spec changes during an experiment.",
+              "format": "int64",
+              "type": "integer"
+            },
+            "id": {
+              "description": "ID is the unique experiment ID sent by Fleet Automation.",
+              "type": "string"
+            },
+            "phase": {
+              "description": "Phase is the current state of the experiment.",
+              "enum": [
+                "running",
+                "stopped",
+                "rollback",
+                "timeout",
+                "promoted",
+                "aborted"
+              ],
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
         "otelAgentGateway": {
           "additionalProperties": false,
           "description": "The actual state of the OTel Agent Gateway as a deployment.",

internal/controller/datadogagent/controller_reconcile_v2_common.go

@@ -826,5 +826,9 @@ func IsEqualStatus(current *v2alpha1.DatadogAgentStatus, newStatus *v2alpha1.Dat
 		return false
 	}
 
+	if !apiequality.Semantic.DeepEqual(current.Experiment, newStatus.Experiment) {
+		return false
+	}
+
 	return condition.IsEqualConditions(current.Conditions, newStatus.Conditions)
 }

internal/controller/datadogagent/controller_reconcile_v2_helpers.go

@@ -259,6 +259,9 @@ func generateNewStatusFromDDA(ddaStatus *datadoghqv2alpha1.DatadogAgentStatus) *
 		if ddaStatus.RemoteConfigConfiguration != nil {
 			status.RemoteConfigConfiguration = ddaStatus.RemoteConfigConfiguration
 		}
+		if ddaStatus.Experiment != nil {
+			status.Experiment = ddaStatus.Experiment.DeepCopy()
+		}
 	}
 	return status
 }