Main kubenest merge

cmd/kubenest/operator/app/operator.go

@@ -27,7 +27,6 @@ import (
 	endpointscontroller "github.com/kosmos.io/kosmos/pkg/kubenest/controller/endpoints.sync.controller"
 	glnodecontroller "github.com/kosmos.io/kosmos/pkg/kubenest/controller/global.node.controller"
 	kosmos "github.com/kosmos.io/kosmos/pkg/kubenest/controller/kosmos"
-	vcnodecontroller "github.com/kosmos.io/kosmos/pkg/kubenest/controller/virtualcluster.node.controller"
 	"github.com/kosmos.io/kosmos/pkg/scheme"
 	"github.com/kosmos.io/kosmos/pkg/sharedcli/klogflag"
 )
@@ -263,15 +262,15 @@ func run(ctx context.Context, config *config.Config) error {
 		return fmt.Errorf("could not create clientset: %v", err)
 	}
 
-	VirtualClusterInitController := controller.VirtualClusterInitController{
-		Client:          mgr.GetClient(),
-		Config:          mgr.GetConfig(),
-		EventRecorder:   mgr.GetEventRecorderFor(constants.InitControllerName),
-		RootClientSet:   hostKubeClient,
-		KosmosClient:    kosmosClient,
-		KubeNestOptions: &config.KubeNestOptions,
-		CoreNamespaces:  config.CoreNamespaces,
-	}
+	VirtualClusterInitController := controller.NewInitController(
+		mgr.GetClient(),
+		mgr.GetConfig(),
+		mgr.GetEventRecorderFor(constants.InitControllerName),
+		hostKubeClient,
+		kosmosClient,
+		&config.KubeNestOptions,
+		config.CoreNamespaces,
+	)
 	if err = VirtualClusterInitController.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error starting %s: %v", constants.InitControllerName, err)
 	}
@@ -299,17 +298,17 @@ func run(ctx context.Context, config *config.Config) error {
 		return err
 	}
 
-	VirtualClusterNodeController := vcnodecontroller.NewNodeController(
-		mgr.GetClient(),
-		hostKubeClient,
-		mgr.GetEventRecorderFor(constants.NodeControllerName),
-		kosmosClient,
-		&config.KubeNestOptions,
-	)
+	// VirtualClusterNodeController := vcnodecontroller.NewNodeController(
+	// 	mgr.GetClient(),
+	// 	hostKubeClient,
+	// 	mgr.GetEventRecorderFor(constants.NodeControllerName),
+	// 	kosmosClient,
+	// 	&config.KubeNestOptions,
+	// )
 
-	if err = VirtualClusterNodeController.SetupWithManager(mgr); err != nil {
-		return fmt.Errorf("error starting %s: %v", constants.NodeControllerName, err)
-	}
+	// if err = VirtualClusterNodeController.SetupWithManager(mgr); err != nil {
+	// 	return fmt.Errorf("error starting %s: %v", constants.NodeControllerName, err)
+	// }
 
 	if config.KubeNestOptions.KubeNestType == v1alpha1.KosmosKube {
 		KosmosJoinController := kosmos.KosmosJoinController{

pkg/kosmosctl/cert/README.md

@@ -0,0 +1,5 @@
+#  update cert for virtual cluster
+
+```
+./kosmosctl renew cert --kubeconfig=config --namespace=test0318 --name=example0318 --agent-user=XXXX --agent-pass=XXXX
+```

pkg/kosmosctl/cert/constant.go

@@ -0,0 +1,41 @@
+package cert
+
+const certShell = `
+#!/usr/bin/env bash
+
+source "env.sh"
+
+CERT_PATH=/apps/conf/kosmos/cert
+
+function update() {
+    echo "exec(1/): copy ca.crt...."
+    cp "$CERT_PATH/ca.crt" "$PATH_KUBERNETES_PKI/ca.crt"
+    if [ $? -ne 0 ]; then
+        exit 1
+    fi
+    echo "exec(2/): copy kubeconfig...."
+    cp "$CERT_PATH/kubelet.conf" "$PATH_KUBERNETES/$KUBELET_KUBE_CONFIG_NAME"
+    if [ $? -ne 0 ]; then
+        exit 1
+    fi
+
+	KUBELET_PKI_PATH="${PATH_KUBELET_LIB}/pki/*"
+    echo "exec(3/): remove pki form kubelet.... ${KUBELET_PKI_PATH}"
+    rm -rf $KUBELET_PKI_PATH
+
+    systemctl restart kubelet
+    systemctl status kubelet 
+
+
+}
+
+# See how we were called.
+case "$1" in
+    update)
+    update
+    ;;
+    *)
+    echo $"usage: $0 update"
+    exit 1
+esac
+`

pkg/kosmosctl/cert/option.go

@@ -0,0 +1,136 @@
+package cert
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"runtime"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sruntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
+	"github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned"
+	"github.com/kosmos.io/kosmos/pkg/scheme"
+)
+
+type Option struct {
+	client.Client
+	remoteClient   clientset.Interface
+	kosmosClient   versioned.Interface
+	virtualCluster *v1alpha1.VirtualCluster
+	dynamicClient  *dynamic.DynamicClient
+	restConfig     *rest.Config
+}
+
+func NewCertOption(o *RenewOptions) (*Option, error) {
+	config, err := clientcmd.BuildConfigFromFlags("", o.KubeconfigPath)
+	if err != nil {
+		klog.Infof("Failed to build config: %v\n", err)
+		return nil, err
+	}
+
+	cli, err := client.New(config, client.Options{Scheme: scheme.NewSchema()})
+	if err != nil {
+		klog.Infof("Failed to create client: %v\n", err)
+		return nil, err
+	}
+
+	dynamicClient, err := dynamic.NewForConfig(config)
+	if err != nil {
+		klog.Infof("Failed to create dynamic client: %v\n", err)
+		return nil, err
+	}
+
+	kosmosClient, err := versioned.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error when creating  kosmosClient client, err: %w", err)
+	}
+
+	localClusterClient, err := clientset.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error when creating local cluster client, err: %w", err)
+	}
+
+	var remoteClient clientset.Interface = localClusterClient
+
+	gvr := schema.GroupVersionResource{
+		Group:    "kosmos.io",
+		Version:  "v1alpha1",
+		Resource: "virtualclusters",
+	}
+
+	unstructuredObj, err := dynamicClient.Resource(gvr).Namespace(o.Namespace).Get(context.TODO(), o.Name, metav1.GetOptions{})
+	if err != nil {
+		fmt.Printf("Failed to get CRD resources: %v\n", err)
+		return nil, err
+	}
+
+	var virtualCluster v1alpha1.VirtualCluster
+	err = k8sruntime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.Object, &virtualCluster)
+	if err != nil {
+		klog.Infof("Error converting to structured object: %v\n", err)
+		return nil, err
+	}
+
+	return &Option{
+		Client:         cli,
+		remoteClient:   remoteClient,
+		kosmosClient:   kosmosClient,
+		virtualCluster: &virtualCluster,
+		dynamicClient:  dynamicClient,
+		restConfig:     config,
+	}, nil
+}
+
+func (c *Option) GetName() string {
+	return c.virtualCluster.GetName()
+}
+
+func (c *Option) GetNamespace() string {
+	return c.virtualCluster.GetNamespace()
+}
+
+func (c *Option) VirtualCluster() *v1alpha1.VirtualCluster {
+	return c.virtualCluster
+}
+
+func (c *Option) DynamicClient() *dynamic.DynamicClient {
+	return c.dynamicClient
+}
+
+func (c *Option) RemoteClient() clientset.Interface {
+	return c.remoteClient
+}
+
+func (c *Option) KosmosClient() versioned.Interface {
+	return c.kosmosClient
+}
+
+func (c *Option) UpdateVirtualCluster(vc *v1alpha1.VirtualCluster) {
+	c.virtualCluster = vc
+}
+
+type TaskFunc func(*Option) error
+
+func getFunctionName(i interface{}) string {
+	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
+}
+func RunTask(tasks []TaskFunc, r *Option) error {
+	total := len(tasks)
+	for index, task := range tasks {
+		klog.Infof("###################### Running task (%d/%d): [%s] \n", index+1, total, getFunctionName(task))
+		err := task(r)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/kosmosctl/cert/renew.go

@@ -1,35 +1,31 @@
 package cert
 
 import (
-	"context"
 	"fmt"
+	"os"
 
 	"github.com/spf13/cobra"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/client-go/dynamic"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	ctlutil "k8s.io/kubectl/pkg/cmd/util"
 	"k8s.io/kubectl/pkg/util/i18n"
 	"k8s.io/kubectl/pkg/util/templates"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
-	"github.com/kosmos.io/kosmos/pkg/kubenest/controller"
-	"github.com/kosmos.io/kosmos/pkg/scheme"
 )
 
 var RenewCertExample = templates.Examples(i18n.T(`
      # Renew cert, e.g:
-     kosmosctl renew cert --kubeconfig=xxxx  --namespace=xxxx --name=xxxx
+     kosmosctl renew cert --kubeconfig=xxxx  --namespace=xxxx --name=xxxx --agent-user=xxxx --agent-pass=xxxx
 `))
 
 type RenewOptions struct {
 	Namespace      string
 	Name           string
 	KubeconfigPath string
+	NodeAgentOptions
+}
+
+type NodeAgentOptions struct {
+	WebUser string
+	WebPass string
 }
 
 func NewCmdRenewCert() *cobra.Command {
@@ -53,7 +49,8 @@ func NewCmdRenewCert() *cobra.Command {
 	flags.StringVarP(&o.Namespace, "namespace", "e", "", "namespace of vc")
 	flags.StringVarP(&o.Name, "name", "n", "", "name of vc")
 	flags.StringVarP(&o.KubeconfigPath, "kubeconfig", "k", "", "kubeconfig path of host cluster")
-
+	flags.StringVarP(&o.WebUser, "agent-user", "u", "", "user of node agent")
+	flags.StringVarP(&o.WebPass, "agent-pass", "p", "", "password of node agent")
 	return cmd
 }
 
@@ -62,60 +59,53 @@ func (o *RenewOptions) Complete() (err error) {
 }
 
 func (o *RenewOptions) Validate() error {
-	return nil
-}
-
-func (o *RenewOptions) Run() error {
-	klog.V(4).Info("kosmos-io.tar.gz has been saved successfully. ")
-
-	config, err := clientcmd.BuildConfigFromFlags("", o.KubeconfigPath)
-	if err != nil {
-		klog.Infof("Failed to build config: %v\n", err)
-		return err
+	if len(o.WebPass) == 0 {
+		return fmt.Errorf("web pass is required")
 	}
 
-	cli, err := client.New(config, client.Options{Scheme: scheme.NewSchema()})
-	if err != nil {
-		klog.Infof("Failed to create client: %v\n", err)
-		return err
+	if len(o.WebUser) == 0 {
+		return fmt.Errorf("use pass is required")
 	}
-
-	dynamicClient, err := dynamic.NewForConfig(config)
-	if err != nil {
-		klog.Infof("Failed to create dynamic client: %v\n", err)
-		return err
+	if len(o.KubeconfigPath) == 0 {
+		return fmt.Errorf("kubeconfig path is required")
 	}
-
-	// 设置 CRD 的 Group、Version、Resource
-	gvr := schema.GroupVersionResource{
-		Group:    "kosmos.io",       // CRD 的 API 组
-		Version:  "v1alpha1",        // CRD 的版本
-		Resource: "virtualclusters", // 资源的复数名称
+	if len(o.Namespace) == 0 {
+		return fmt.Errorf("namespace is required")
 	}
-
-	// 获取 CRD 资源
-	unstructuredObj, err := dynamicClient.Resource(gvr).Namespace(o.Namespace).Get(context.TODO(), o.Name, metav1.GetOptions{})
-	if err != nil {
-		fmt.Printf("Failed to get CRD resources: %v\n", err)
-		return err
+	if len(o.Name) == 0 {
+		return fmt.Errorf("name is required")
 	}
+	return nil
+}
 
-	var virtualCluster v1alpha1.VirtualCluster
-	err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.Object, &virtualCluster)
-	if err != nil {
-		klog.Infof("Error converting to structured object: %v\n", err)
-		return err
-	}
+func (o *RenewOptions) initEnv() {
+	os.Setenv("KUBECONFIG", o.KubeconfigPath)
+	os.Setenv("WEB_USER", o.WebUser)
+	os.Setenv("WEB_PASS", o.WebPass)
+}
 
-	exec, err := controller.UpdateCertPhase(&virtualCluster, cli, config, &v1alpha1.KubeNestConfiguration{})
+func (o *RenewOptions) Run() error {
+	r, err := NewCertOption(o)
+	o.initEnv()
 	if err != nil {
-		panic(err)
+		return err
 	}
+	return Do(r)
+}
 
-	err = exec.Execute()
+func Do(r *Option) error {
+	err := RunTask([]TaskFunc{
+		RunCheckEnvironment,
+		RunBackupSecrets,
+		RunReCreateCertAndKubeConfig,
+		UpdateKubeProxyConfig,
+		RestartVirtualControlPlanePod,
+		RestartVirtualWorkerKubelet,
+		RestartVirtualPod,
+	}, r)
 	if err != nil {
-		panic(err)
+		return err
 	}
-
+	klog.Infof("############ renew cert success!!!!")
 	return nil
 }