feat: add webhook for change imagemanager ref which bind dev container (#13)

Author: hysyeah

cmd/devbox/main.go

@@ -80,6 +80,7 @@ func main() {
 			config := ctrl.GetConfigOrDie()
 			wh := webhook.Webhook{KubeClient: kubernetes.NewForConfigOrDie(config)}
 			runtime.Must(wh.DeleteDevContainerMutatingWebhook())
+			runtime.Must(wh.DeleteImageManagerMutatingWebhook())
 		},
 	}
 

pkg/api/server/handlers_api.go

@@ -3,7 +3,6 @@ package server
 import (
 	"errors"
 	"fmt"
-	"github.com/go-resty/resty/v2"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -17,6 +16,7 @@ import (
 	"github.com/beclab/devbox/pkg/store/db/model"
 
 	"github.com/emicklei/go-restful/v3"
+	"github.com/go-resty/resty/v2"
 	"github.com/gofiber/fiber/v2"
 	"github.com/google/uuid"
 	"gopkg.in/yaml.v2"
@@ -194,6 +194,7 @@ func (h *handlers) bindContainer(ctx *fiber.Ctx) error {
 		AppId            int     `json:"appId"`
 		PodSelector      string  `json:"podSelector"`
 		ContainerName    string  `json:"containerName"`
+		Image            string  `json:"image"`
 		DevEnv           *string `json:"devEnv,omitempty"`
 		DevContainerName string  `json:"devContainerName"`
 	}
@@ -279,6 +280,7 @@ func (h *handlers) bindContainer(ctx *fiber.Ctx) error {
 		ContainerID:   uint(containerId),
 		PodSelector:   postData.PodSelector,
 		ContainerName: postData.ContainerName,
+		Image:         postData.Image,
 	}
 
 	err = h.db.DB.Create(&appContainer).Error

pkg/api/server/handlers_webhook.go

@@ -3,9 +3,9 @@ package server
 import (
 	"context"
 	"fmt"
-
-	"github.com/beclab/devbox/pkg/webhook"
 
+	"github.com/beclab/devbox/pkg/webhook"
+
 	"github.com/gofiber/fiber/v2"
 	"github.com/google/uuid"
 	admissionv1 "k8s.io/api/admission/v1"
@@ -87,3 +87,46 @@ func (h *webhooks) mutate(ctx context.Context, req *admissionv1.AdmissionRequest
 	return resp
 
 }
+
+func (h *webhooks) imageManager(ctx *fiber.Ctx) error {
+	klog.Infof("Received mutating webhook request: Method=%v, URL=%v", ctx.Method(), ctx.OriginalURL())
+	admissionRequestBody := ctx.BodyRaw()
+	if len(admissionRequestBody) == 0 {
+		klog.Error("Error reading admission request body, body is empty")
+		return fiber.NewError(fiber.StatusBadRequest, "empty request admission request body")
+	}
+	var admissionReq, admissionResp admissionv1.AdmissionReview
+	proxyUUID := uuid.New()
+	if _, _, err := webhook.Deserializer.Decode(admissionRequestBody, nil, &admissionReq); err != nil {
+		klog.Error("Error decoding admission request body, ", err)
+		admissionResp.Response = h.webhook.AdmissionError(err)
+	} else {
+		admissionResp.Response = h.imageManagerMutate(ctx.Context(), admissionReq.Request, proxyUUID)
+	}
+
+	admissionResp.TypeMeta = admissionReq.TypeMeta
+	admissionResp.Kind = admissionReq.Kind
+
+	return ctx.JSON(&admissionResp)
+}
+
+func (h *webhooks) imageManagerMutate(ctx context.Context, req *admissionv1.AdmissionRequest, proxyUUID uuid.UUID) *admissionv1.AdmissionResponse {
+	if req == nil {
+		klog.Error("nil admission Request")
+		return h.webhook.AdmissionError(errNilAdmissionRequest)
+	}
+	resp := &admissionv1.AdmissionResponse{
+		Allowed: true,
+		UID:     req.UID,
+	}
+
+	klog.Info("Creating patch for resource ", req.Resource)
+	patchBytes, err := h.webhook.MutateIm(ctx, req.Object.Raw, proxyUUID)
+	if err != nil {
+		return h.webhook.AdmissionError(err)
+	}
+	if len(patchBytes) > 0 {
+		h.webhook.PatchAdmissionResponse(resp, patchBytes)
+	}
+	return resp
+}

pkg/api/server/server.go

@@ -36,6 +36,7 @@ func NewServer(db *db.DbOperator) *server {
 		DB:         db,
 	}
 	utilruntime.Must(webhook.CreateOrUpdateDevContainerMutatingWebhook())
+	utilruntime.Must(webhook.CreateOrUpdateImageManagerMutatingWebhook())
 
 	return &server{
 		handlers: &handlers{db: db, kubeConfig: config},
@@ -104,6 +105,7 @@ func (s *server) Start() {
 	// webhooks /webhook
 	wh := webhookServer.Group("webhook")
 	wh.Post("/devcontainer", s.webhooks.devcontainer)
+	wh.Post("/imagemanager", s.webhooks.imageManager)
 
 	klog.Info("dev box api server listening on 8088 ")
 

pkg/store/db/model/dev_app_container.go

@@ -8,6 +8,7 @@ type DevAppContainers struct {
 	ContainerID   uint      `gorm:"column:container_id" json:"containerId"`
 	PodSelector   string    `gorm:"type:varchar(50);column:pod_selector" json:"podSelector"`
 	ContainerName string    `gorm:"type:varchar(50);column:container_name" json:"containerName"`
+	Image         string    `gorm:"type:varchar(128);column:image" json:"image"`
 	CreateTime    time.Time `gorm:"default:CURRENT_TIMESTAMP;column:create_time" json:"createTime"`
 	UpdateTime    time.Time `gorm:"default:CURRENT_TIMESTAMP;column:update_time" json:"updateTime"`
 

pkg/store/db/model/dev_container.go

@@ -19,6 +19,7 @@ type DevContainerInfo struct {
 	PodSelector   *string `json:"podSelector,omitempty"`
 	AppID         *int    `json:"appId,omitempty"`
 	ContainerName *string `json:"containerName,omitempty"`
+	Image         *string `json:"image,omitempty"`
 	AppName       *string `json:"appName,omitempty"`
 	State         *string `json:"state,omitempty"`
 	DevPath       *string `json:"devPath,omitempty"`

pkg/store/db/operator.go

@@ -62,6 +62,13 @@ func createTableIfNotExists() (err error) {
 		if err != nil {
 			return err
 		}
+	} else {
+		if !db.Migrator().HasColumn(&model.DevAppContainers{}, "Image") {
+			err = db.Migrator().AddColumn(&model.DevAppContainers{}, "Image")
+			if err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }

pkg/webhook/funcs.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -13,14 +14,17 @@ import (
 	"github.com/beclab/devbox/pkg/development/container"
 	"github.com/beclab/devbox/pkg/development/envoy"
 	"github.com/beclab/devbox/pkg/development/helm"
+	"github.com/beclab/devbox/pkg/store/db"
 	"github.com/beclab/devbox/pkg/store/db/model"
 
+	"github.com/containerd/containerd/reference/docker"
 	"github.com/google/uuid"
 	"gorm.io/gorm"
 	admissionv1 "k8s.io/api/admission/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/klog/v2"
@@ -493,3 +497,59 @@ func (wh *Webhook) getUserHomeDir(ctx context.Context) (string, error) {
 	}
 	return filepath.Join(dir, "Home"), nil
 }
+
+func (wh *Webhook) MutateIm(ctx context.Context, raw []byte, proxyUUID uuid.UUID) (patch []byte, err error) {
+	var obj unstructured.Unstructured
+	if err := json.Unmarshal(raw, &obj); err != nil {
+		klog.Errorf("Error unmarshaling request to unstructured err=%v", err)
+		return nil, err
+	}
+	appName, _, _ := unstructured.NestedString(obj.Object, "spec", "appName")
+	originAppName := strings.TrimSuffix(appName, "-dev")
+	refs, _, _ := unstructured.NestedSlice(obj.Object, "spec", "refs")
+
+	sql := `select dc.id,dc.dev_env,dc.name, dc.create_time, dc.update_time, ac.pod_selector,ac.app_id, ac.container_name,ac.image,a.app_name
+	from dev_apps a
+	join dev_app_containers ac on a.id = ac.app_id
+	join dev_containers dc on ac.container_id = dc.id
+	where app_name = '%s'`
+	sql = fmt.Sprintf(sql, originAppName)
+	list := make([]*model.DevContainerInfo, 0)
+	db := db.NewDbOperator()
+	err = db.DB.Raw(sql).Scan(&list).Error
+	if err != nil {
+		return nil, err
+	}
+	klog.Infof("len(list)=%v", len(list))
+	if len(list) == 0 {
+		return makePatches(raw, obj.Object, appName)
+	}
+
+	newRefs := make([]interface{}, 0)
+	for _, r := range refs {
+		name := r.(map[string]interface{})["name"].(string)
+		pullPolicy := r.(map[string]interface{})["imagePullPolicy"].(string)
+		image, _ := docker.ParseDockerRef(*list[0].Image)
+
+		devImage, _ := docker.ParseDockerRef(container.DevEnvImage(list[0].DevEnv))
+		if name == image.String() {
+			name = devImage.String()
+		}
+		newRefs = append(newRefs, map[string]interface{}{
+			"name":            name,
+			"imagePullPolicy": pullPolicy,
+		})
+	}
+	klog.Infof("newRefs: %#v", newRefs)
+	err = unstructured.SetNestedSlice(obj.Object, newRefs, "spec", "refs")
+	if err != nil {
+		return nil, err
+	}
+
+	patch, err = makePatches(raw, obj.Object, appName)
+	if err != nil {
+		klog.Infof("make Patches err=%v", err)
+	}
+	klog.Infof("pathc: %v", string(patch))
+	return patch, err
+}

pkg/webhook/setup.go

@@ -21,6 +21,8 @@ const (
 	defaultCaPath              = "/etc/certs/ca.crt"
 	webhookServiceName         = "devbox-server"
 	devContainerWebhookCfgName = "devcontainer-mutate-webhooks"
+	imageManagerWebhookCfgName = "imagemanager-mutate-webhooks"
+	imageManagerWebhookPrefix  = "imagemanager-webhook"
 	mutatingWebhookNamePrefix  = "devcontainer-webhook"
 	helmRelease                = "meta.helm.sh/release-name"
 	helmReleaseNamespace       = "meta.helm.sh/release-namespace"
@@ -31,6 +33,8 @@ var (
 	webhookServiceNamespace       = &constants.Namespace
 	webhookPath                   = "/webhook/devcontainer"
 	WebhookPort             int32 = 8083
+
+	imageManagerWebhookPath = "/webhook/imagemanager"
 	// WebhookServerListenAddress       = webhookServiceName + ":" + strconv.Itoa(int(WebhookPort))
 
 	// codecs is the codec factory used by the deserialzer
@@ -202,7 +206,150 @@ func (wh *Webhook) DeleteDevContainerMutatingWebhook() error {
 	return nil
 }
 
+func (wh *Webhook) CreateOrUpdateImageManagerMutatingWebhook() error {
+	failurePolicy := admissionregv1.Fail
+	matchPolicy := admissionregv1.Exact
+	webhookTimeout := int32(30)
+
+	caBundle, err := os.ReadFile(defaultCaPath)
+	if err != nil {
+		return err
+	}
+
+	mwhLabels := map[string]string{"velero.io/exclude-from-backup": "true"}
+	mwh := admissionregv1.MutatingWebhookConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   imageManagerWebhookCfgName,
+			Labels: mwhLabels,
+		},
+		Webhooks: []admissionregv1.MutatingWebhook{},
+	}
+	imwh := admissionregv1.MutatingWebhook{
+		Name: imageManagerWebhookName(),
+		ClientConfig: admissionregv1.WebhookClientConfig{
+			CABundle: caBundle,
+			Service: &admissionregv1.ServiceReference{
+				Namespace: *webhookServiceNamespace,
+				Name:      webhookServiceName,
+				Path:      &imageManagerWebhookPath,
+				Port:      &WebhookPort,
+			},
+		},
+		FailurePolicy: &failurePolicy,
+		MatchPolicy:   &matchPolicy,
+		Rules: []admissionregv1.RuleWithOperations{
+			{
+				Operations: []admissionregv1.OperationType{admissionregv1.Create},
+				Rule: admissionregv1.Rule{
+					APIGroups:   []string{"app.bytetrade.io"},
+					APIVersions: []string{"*"},
+					Resources:   []string{"imagemanagers"},
+				},
+			},
+		},
+		ObjectSelector: &metav1.LabelSelector{
+			MatchExpressions: []metav1.LabelSelectorRequirement{
+				{
+					Key:      constants.DevOwnerLabel,
+					Operator: metav1.LabelSelectorOpIn,
+					Values:   []string{constants.Owner},
+				},
+			},
+		},
+		SideEffects: func() *admissionregv1.SideEffectClass {
+			sideEffect := admissionregv1.SideEffectClassNoneOnDryRun
+			return &sideEffect
+		}(),
+		TimeoutSeconds:          &webhookTimeout,
+		AdmissionReviewVersions: []string{"v1"},
+	}
+	mwh.Webhooks = append(mwh.Webhooks, imwh)
+	if _, err = wh.KubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), &mwh, metav1.CreateOptions{}); err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+				existing, err := wh.KubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), mwh.Name, metav1.GetOptions{})
+				if err != nil {
+					klog.Error("Error getting MutatingWebhookConfiguration ", err)
+					return err
+				}
+				found := false
+				for i, w := range existing.Webhooks {
+					if w.Name == imwh.Name {
+						found = true
+						existing.Webhooks[i] = imwh
+						break
+					}
+				}
+				if !found {
+					existing.Webhooks = append(existing.Webhooks, imwh)
+				}
+				_, err = wh.KubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), existing, metav1.UpdateOptions{})
+				if err != nil && !apierrors.IsConflict(err) {
+					klog.Error("Error updating MutatingWebhookConfiguration ", err)
+				}
+				return err
+			})
+			if err != nil {
+				klog.Error("Error updating MutatingWebhookConfiguration ", err)
+				return err
+			}
+		} else {
+			klog.Error("Error creating MutatingWebhookConfiguration ", err)
+			return err
+		}
+	}
+	klog.Infof("Finished creating MutatingWebhookConfiguration %s", imageManagerWebhookCfgName)
+	return nil
+}
+
+func (wh *Webhook) DeleteImageManagerMutatingWebhook() error {
+	imwhName := imageManagerWebhookName()
+	existing, err := wh.KubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), imageManagerWebhookCfgName, metav1.GetOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			klog.Info("webhook configuration not found, ", imageManagerWebhookCfgName)
+			return nil
+		}
+		return err
+	}
+	for i, w := range existing.Webhooks {
+		if w.Name == imwhName {
+			if len(existing.Webhooks) == 1 {
+				err = wh.KubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), imageManagerWebhookCfgName, metav1.DeleteOptions{})
+				if err != nil {
+					klog.Info("delete webhook configuration error, ", err)
+					return err
+				}
+			} else {
+				return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+					updating, err := wh.KubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), imageManagerWebhookCfgName, metav1.GetOptions{})
+					if err != nil {
+						if apierrors.IsNotFound(err) {
+							klog.Info("webhook configuration not found, ", imageManagerWebhookCfgName)
+							return nil
+						}
+						return err
+					}
+					updating.Webhooks = append(existing.Webhooks[:i], existing.Webhooks[i+1:]...)
+					klog.Info("removing the webhook, ", imwhName)
+					_, err = wh.KubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.Background(), updating, metav1.UpdateOptions{})
+					if !apierrors.IsConflict(err) {
+						klog.Error("Error updating MutatingWebhookConfiguration ", err)
+					}
+					return err
+				})
+			}
+		}
+	}
+	klog.Infof("success to clean imagemanager webhook")
+	return nil
+}
+
 func mutatingWebhookName() string {
 	// should be a domain with at least three segments separated by dots
 	return mutatingWebhookNamePrefix + "." + constants.Namespace + ".ns"
 }
+
+func imageManagerWebhookName() string {
+	return imageManagerWebhookPrefix + "." + constants.Namespace + ".ns"
+}