diff --git a/Makefile b/Makefile index eac91608..aec0625a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,10 @@ -all: interlink vk installer ssh-tunnel +all: interlink interlink-cri vk installer ssh-tunnel interlink: - CGO_ENABLED=0 OOS=linux go build -o bin/interlink cmd/interlink/main.go cmd/interlink/cri.go + CGO_ENABLED=0 OOS=linux go build -o bin/interlink cmd/interlink/main.go + +interlink-cri: + CGO_ENABLED=0 OOS=linux go build -o bin/interlink-cri cmd/interlink-cri/*.go vk: CGO_ENABLED=0 OOS=linux go build -o bin/vk cmd/virtual-kubelet/main.go diff --git a/ci/main.go b/ci/main.go index 2750fae6..ec34e494 100644 --- a/ci/main.go +++ b/ci/main.go @@ -647,7 +647,7 @@ func (m *Interlink) BuildImages( WithEnvVariable("GOCACHE", "/go/build-cache"). WithEnvVariable("CGO_ENABLED", "0"). WithExec([]string{"bash", "-c", "KUBELET_VERSION=${VERSION} ./cmd/virtual-kubelet/set-version.sh"}). - WithExec([]string{"go", "build", "-o", "bin/interlink", "cmd/interlink/main.go", "cmd/interlink/cri.go"}) + WithExec([]string{"go", "build", "-o", "bin/interlink", "cmd/interlink/main.go"}) m.InterlinkContainer = dag.Container(). From("alpine"). diff --git a/cmd/interlink/cri.go b/cmd/interlink-cri/cri.go similarity index 50% rename from cmd/interlink/cri.go rename to cmd/interlink-cri/cri.go index 90719bf2..3ce3b5b1 100644 --- a/cmd/interlink/cri.go +++ b/cmd/interlink-cri/cri.go @@ -1,35 +1,235 @@ package main import ( + "bytes" "context" + "crypto/tls" + "crypto/x509" + "encoding/json" "fmt" + "io" + "math/rand" + "net/http" + "os" + "strings" "time" + "github.com/containerd/containerd/log" + types "github.com/interlink-hq/interlink/pkg/interlink" apitest "github.com/interlink-hq/interlink/pkg/interlink/cri" + vkconfig "github.com/interlink-hq/interlink/pkg/virtualkubelet" "google.golang.org/grpc" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ktypes "k8s.io/apimachinery/pkg/types" kubeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/cri-client/pkg/util" utilexec "k8s.io/utils/exec" ) -// RemoteRuntime represents a fake remote container runtime. +// InterLink API functions for CRI integration (similar to virtual-kubelet execute.go pattern) + +// CRI-specific session context management (similar to virtual-kubelet) +func addSessionContext(req *http.Request, sessionContext string) { + req.Header.Set("X-Session-Context", sessionContext) +} + +// createTLSHTTPClient creates an HTTP client with TLS/mTLS configuration (from virtual-kubelet pattern) +func createTLSHTTPClient(ctx context.Context, tlsConfig vkconfig.TLSConfig) (*http.Client, error) { + if !tlsConfig.Enabled { + return http.DefaultClient, nil + } + + transport := &http.Transport{ + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + }, + } + + // Load CA certificate if provided + if tlsConfig.CACertFile != "" { + caCert, err := os.ReadFile(tlsConfig.CACertFile) + if err != nil { + return nil, fmt.Errorf("failed to read CA certificate file %s: %w", tlsConfig.CACertFile, err) + } + + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to parse CA certificate from %s", tlsConfig.CACertFile) + } + transport.TLSClientConfig.RootCAs = caCertPool + log.G(ctx).Info("Loaded CA certificate for TLS client from: ", tlsConfig.CACertFile) + } + + // Load client certificate and key for mTLS if provided + if tlsConfig.CertFile != "" && tlsConfig.KeyFile != "" { + cert, err := tls.LoadX509KeyPair(tlsConfig.CertFile, tlsConfig.KeyFile) + if err != nil { + return nil, fmt.Errorf("failed to load client certificate pair (%s, %s): %w", tlsConfig.CertFile, tlsConfig.KeyFile, err) + } + transport.TLSClientConfig.Certificates = []tls.Certificate{cert} + log.G(ctx).Info("Loaded client certificate for mTLS from: ", tlsConfig.CertFile, " and ", tlsConfig.KeyFile) + } + + return &http.Client{Transport: transport}, nil +} + +// getInterlinkEndpoint constructs the interLink API endpoint +func getInterlinkEndpoint(ctx context.Context, interlinkURL string, interlinkPort string) string { + interlinkEndpoint := "" + log.G(ctx).Info("InterlinkURL: ", interlinkURL) + switch { + case strings.HasPrefix(interlinkURL, "unix://"): + interlinkEndpoint = "http://unix" + case strings.HasPrefix(interlinkURL, "http://"): + interlinkEndpoint = interlinkURL + ":" + interlinkPort + case strings.HasPrefix(interlinkURL, "https://"): + interlinkEndpoint = interlinkURL + ":" + interlinkPort + default: + log.G(ctx).Fatal("InterLinkURL should either start with unix:// or http(s)://") + } + return interlinkEndpoint +} + +// doRequestWithClient performs HTTP request with authentication (similar to virtual-kubelet) +func doRequestWithClient(req *http.Request, token string, httpClient *http.Client) (*http.Response, error) { + if token != "" { + req.Header.Add("Authorization", "Bearer "+token) + } + req.Header.Set("Content-Type", "application/json") + return httpClient.Do(req) +} + +// createPodRequest performs a REST call to the InterLink API to create a pod +func (f *RemoteRuntime) createPodRequest(ctx context.Context, pod types.PodCreateRequests) ([]byte, error) { + interlinkEndpoint := getInterlinkEndpoint(ctx, f.Config.InterlinkURL, f.Config.InterlinkPort) + + bodyBytes, err := json.Marshal(pod) + if err != nil { + log.G(ctx).Error(err) + return nil, err + } + + reader := bytes.NewReader(bodyBytes) + req, err := http.NewRequest(http.MethodPost, interlinkEndpoint+"/create", reader) + if err != nil { + log.G(ctx).Error(err) + return nil, err + } + + // Add session context for tracing + sessionContext := fmt.Sprintf("CRI-CreatePod#%d", rand.Intn(100000)) + addSessionContext(req, sessionContext) + + // Get token if configured + token := "" + if f.Config.VKTokenFile != "" { + tokenBytes, err := os.ReadFile(f.Config.VKTokenFile) + if err != nil { + log.G(ctx).Error(err) + return nil, err + } + token = string(tokenBytes) + } + + // Create TLS-enabled HTTP client + httpClient, err := createTLSHTTPClient(ctx, f.Config.TLS) + if err != nil { + return nil, fmt.Errorf("failed to create TLS HTTP client: %w", err) + } + + resp, err := doRequestWithClient(req, token, httpClient) + if err != nil { + return nil, fmt.Errorf("error doing doRequest() in createPodRequest(): %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected error creating pod. Status code: %d", resp.StatusCode) + } + + return io.ReadAll(resp.Body) +} + +// deletePodRequest performs a REST call to the InterLink API to delete a pod +func (f *RemoteRuntime) deletePodRequest(ctx context.Context, pod *v1.Pod) error { + interlinkEndpoint := getInterlinkEndpoint(ctx, f.Config.InterlinkURL, f.Config.InterlinkPort) + + bodyBytes, err := json.Marshal(pod) + if err != nil { + log.G(ctx).Error(err) + return err + } + + reader := bytes.NewReader(bodyBytes) + req, err := http.NewRequest(http.MethodDelete, interlinkEndpoint+"/delete", reader) + if err != nil { + log.G(ctx).Error(err) + return err + } + + // Add session context for tracing + sessionContext := fmt.Sprintf("CRI-DeletePod#%d", rand.Intn(100000)) + addSessionContext(req, sessionContext) + + // Get token if configured + token := "" + if f.Config.VKTokenFile != "" { + tokenBytes, err := os.ReadFile(f.Config.VKTokenFile) + if err != nil { + log.G(ctx).Error(err) + return err + } + token = string(tokenBytes) + } + + // Create TLS-enabled HTTP client + httpClient, err := createTLSHTTPClient(ctx, f.Config.TLS) + if err != nil { + return fmt.Errorf("failed to create TLS HTTP client: %w", err) + } + + resp, err := doRequestWithClient(req, token, httpClient) + if err != nil { + return fmt.Errorf("error doing doRequest() in deletePodRequest(): %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected error deleting pod. Status code: %d", resp.StatusCode) + } + + return nil +} + +// RemoteRuntime represents a remote container runtime that integrates with interLink server API. type RemoteRuntime struct { server *grpc.Server - // Fake runtime service. + // Fake runtime service for CRI compatibility RuntimeService *apitest.FakeRuntimeService - // Fake image service. + // Fake image service for CRI compatibility ImageService *apitest.FakeImageService + // Configuration for interLink API communication (like virtual-kubelet) + Config vkconfig.Config + // Context for the runtime + Ctx context.Context + // Container-to-Pod mapping for tracking + ContainerPodMap map[string]string // containerID -> podUID } -// NewFakeRemoteRuntime creates a new RemoteRuntime. -func NewFakeRemoteRuntime() *RemoteRuntime { +// NewFakeRemoteRuntime creates a new RemoteRuntime with InterLink server API integration. +func NewFakeRemoteRuntime(ctx context.Context, config vkconfig.Config) *RemoteRuntime { fakeRuntimeService := apitest.NewFakeRuntimeService() fakeImageService := apitest.NewFakeImageService() f := &RemoteRuntime{ - server: grpc.NewServer(), - RuntimeService: fakeRuntimeService, - ImageService: fakeImageService, + server: grpc.NewServer(), + RuntimeService: fakeRuntimeService, + ImageService: fakeImageService, + Config: config, + Ctx: ctx, + ContainerPodMap: make(map[string]string), } kubeapi.RegisterRuntimeServiceServer(f.server, f) kubeapi.RegisterImageServiceServer(f.server, f.ImageService) @@ -96,11 +296,46 @@ func (f *RemoteRuntime) StopPodSandbox(ctx context.Context, req *kubeapi.StopPod return &kubeapi.StopPodSandboxResponse{}, nil } -// RemovePodSandbox removes the sandbox. If there are any running containers -// in the sandbox, they must be forcibly terminated and removed. -// This call is idempotent, and must not return an error if the sandbox has -// already been removed. +// RemovePodSandbox removes the sandbox using interLink lifecycle func (f *RemoteRuntime) RemovePodSandbox(ctx context.Context, req *kubeapi.RemovePodSandboxRequest) (*kubeapi.RemovePodSandboxResponse, error) { + // Check if we have a pod to delete via interLink API + // Find any container mapped to this sandbox to get pod info + var podUID string + for containerID, uid := range f.ContainerPodMap { + f.RuntimeService.Lock() + container, exists := f.RuntimeService.Containers[containerID] + f.RuntimeService.Unlock() + + if exists && container.SandboxID == req.PodSandboxId { + podUID = uid + break + } + } + + if podUID != "" { + // Create a minimal pod object for deletion + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + UID: ktypes.UID(podUID), + }, + } + + // Call interLink delete API + err := f.deletePodRequest(ctx, pod) + if err != nil { + log.G(ctx).Warning("Failed to delete pod via interLink API: ", err) + } else { + log.G(ctx).Info("Pod deleted successfully via interLink API: ", podUID) + } + + // Clean up our container mappings for this sandbox + for containerID, uid := range f.ContainerPodMap { + if uid == podUID { + delete(f.ContainerPodMap, containerID) + } + } + } + err := f.RuntimeService.StopPodSandbox(ctx, req.PodSandboxId) if err != nil { return nil, err @@ -130,43 +365,129 @@ func (f *RemoteRuntime) ListPodSandbox(ctx context.Context, req *kubeapi.ListPod return &kubeapi.ListPodSandboxResponse{Items: items}, nil } -// CreateContainer creates a new container in specified PodSandbox +// convertToInterLinkPod converts CRI container config to interLink pod format +func (f *RemoteRuntime) convertToInterLinkPod(_ context.Context, sandboxConfig *kubeapi.PodSandboxConfig, containerConfig *kubeapi.ContainerConfig) *v1.Pod { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: sandboxConfig.Metadata.Name, + Namespace: sandboxConfig.Metadata.Namespace, + UID: ktypes.UID(sandboxConfig.Metadata.Uid), + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: containerConfig.Metadata.Name, + Image: containerConfig.Image.Image, + Command: containerConfig.Command, + Args: containerConfig.Args, + WorkingDir: containerConfig.WorkingDir, + }, + }, + }, + Status: v1.PodStatus{ + Phase: v1.PodPending, + }, + } + + // Convert environment variables + if len(containerConfig.Envs) > 0 { + envVars := make([]v1.EnvVar, len(containerConfig.Envs)) + for i, env := range containerConfig.Envs { + envVars[i] = v1.EnvVar{ + Name: env.Key, + Value: env.Value, + } + } + pod.Spec.Containers[0].Env = envVars + } + + // Convert resource requirements + if containerConfig.Linux != nil && containerConfig.Linux.Resources != nil { + resources := v1.ResourceRequirements{} + if containerConfig.Linux.Resources.CpuQuota > 0 { + resources.Limits = v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(containerConfig.Linux.Resources.CpuQuota/1000, resource.DecimalSI), + } + } + if containerConfig.Linux.Resources.MemoryLimitInBytes > 0 { + if resources.Limits == nil { + resources.Limits = v1.ResourceList{} + } + resources.Limits[v1.ResourceMemory] = *resource.NewQuantity(containerConfig.Linux.Resources.MemoryLimitInBytes, resource.BinarySI) + } + pod.Spec.Containers[0].Resources = resources + } + + return pod +} + +// CreateContainer creates a new container in specified PodSandbox using interLink lifecycle func (f *RemoteRuntime) CreateContainer(ctx context.Context, req *kubeapi.CreateContainerRequest) (*kubeapi.CreateContainerResponse, error) { + // First create the container using the fake runtime for CRI compatibility containerID, err := f.RuntimeService.CreateContainer(ctx, req.PodSandboxId, req.Config, req.SandboxConfig) if err != nil { return nil, err } + // Integrate with interLink server API + // Convert CRI config to interLink pod format + pod := f.convertToInterLinkPod(ctx, req.SandboxConfig, req.Config) + + // Create pod creation request for interLink + podCreateReq := types.PodCreateRequests{ + Pod: *pod, + } + + // Call interLink server API to create the pod + _, err = f.createPodRequest(ctx, podCreateReq) + if err != nil { + log.G(ctx).Warning("Failed to create pod via interLink API: ", err) + // Continue with CRI response even if interLink fails + } else { + // Store container to pod mapping for lifecycle tracking + f.ContainerPodMap[containerID] = string(pod.UID) + log.G(ctx).Info("Pod created successfully via interLink API: ", pod.Name) + } + return &kubeapi.CreateContainerResponse{ContainerId: containerID}, nil } -// StartContainer starts the container. +// StartContainer starts the container using interLink lifecycle func (f *RemoteRuntime) StartContainer(ctx context.Context, req *kubeapi.StartContainerRequest) (*kubeapi.StartContainerResponse, error) { + // Start container in fake runtime for CRI compatibility err := f.RuntimeService.StartContainer(ctx, req.ContainerId) if err != nil { return nil, err } + // Container start is handled by interLink server API + // The pod was already created during CreateContainer, so starting is managed by the plugin + log.G(ctx).Info("Container started via CRI: ", req.ContainerId) + return &kubeapi.StartContainerResponse{}, nil } -// StopContainer stops a running container with a grace period (i.e., timeout). -// This call is idempotent, and must not return an error if the container has -// already been stopped. +// StopContainer stops a running container using interLink lifecycle func (f *RemoteRuntime) StopContainer(ctx context.Context, req *kubeapi.StopContainerRequest) (*kubeapi.StopContainerResponse, error) { + // Stop container in fake runtime for CRI compatibility err := f.RuntimeService.StopContainer(ctx, req.ContainerId, req.Timeout) if err != nil { return nil, err } + // Container stop is handled by interLink server API + // The interLink plugin manages the container lifecycle + log.G(ctx).Info("Container stopped via CRI: ", req.ContainerId) + return &kubeapi.StopContainerResponse{}, nil } -// RemoveContainer removes the container. If the container is running, the -// container must be forcibly removed. -// This call is idempotent, and must not return an error if the container has -// already been removed. +// RemoveContainer removes the container using interLink lifecycle func (f *RemoteRuntime) RemoveContainer(ctx context.Context, req *kubeapi.RemoveContainerRequest) (*kubeapi.RemoveContainerResponse, error) { + // Clean up container-to-pod mapping + delete(f.ContainerPodMap, req.ContainerId) + + // Remove container from fake runtime err := f.RuntimeService.RemoveContainer(ctx, req.ContainerId) if err != nil { return nil, err @@ -185,14 +506,18 @@ func (f *RemoteRuntime) ListContainers(ctx context.Context, req *kubeapi.ListCon return &kubeapi.ListContainersResponse{Containers: items}, nil } -// ContainerStatus returns status of the container. If the container is not -// present, returns an error. +// ContainerStatus returns status of the container using interLink lifecycle data func (f *RemoteRuntime) ContainerStatus(ctx context.Context, req *kubeapi.ContainerStatusRequest) (*kubeapi.ContainerStatusResponse, error) { resp, err := f.RuntimeService.ContainerStatus(ctx, req.ContainerId, false) if err != nil { return nil, err } + // Optionally get status from interLink server API + // For now, we rely on the fake runtime status since it provides CRI compatibility + // Future enhancement: query interLink status API for real container states + log.G(ctx).Debug("Container status requested for: ", req.ContainerId) + return resp, nil } diff --git a/cmd/interlink-cri/main.go b/cmd/interlink-cri/main.go new file mode 100644 index 00000000..3f8fd45b --- /dev/null +++ b/cmd/interlink-cri/main.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/sirupsen/logrus" + "github.com/virtual-kubelet/virtual-kubelet/log" + logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus" + "github.com/virtual-kubelet/virtual-kubelet/trace" + "github.com/virtual-kubelet/virtual-kubelet/trace/opentelemetry" + "k8s.io/cri-client/pkg/util" + + "github.com/interlink-hq/interlink/pkg/interlink" + vkconfig "github.com/interlink-hq/interlink/pkg/virtualkubelet" +) + +const ( + CRIVersion = "0.6.0-dev" + trueValue = "true" +) + +func main() { + var ( + printVersion = flag.Bool("version", false, "show version") + configPath = flag.String("config", "", "path to configuration file") + socketPath = flag.String("socket", "unix:///var/run/interlink/cri.sock", "CRI socket path") + ) + flag.Parse() + + if *printVersion { + fmt.Printf("InterLink CRI %s\n", CRIVersion) + return + } + + // Load configuration + var criConfig vkconfig.Config + if *configPath != "" { + // Load from file (implement config loading similar to interLink) + log.L.Info("Loading CRI configuration from: ", *configPath) + // For now, use environment variables and defaults + } + + // Set up from environment variables if config not provided + if criConfig.InterlinkURL == "" { + criConfig.InterlinkURL = getEnvOrDefault("INTERLINK_URL", "https://localhost:8080") + } + if criConfig.InterlinkPort == "" { + criConfig.InterlinkPort = getEnvOrDefault("INTERLINK_PORT", "8080") + } + criConfig.VKTokenFile = os.Getenv("VK_TOKEN_FILE") + criConfig.VerboseLogging = getEnvOrDefault("INTERLINK_CRI_VERBOSE", "false") == trueValue + criConfig.ErrorsOnlyLogging = getEnvOrDefault("INTERLINK_CRI_ERRORS_ONLY", "false") == trueValue + + // TLS configuration from environment + criConfig.TLS.Enabled = getEnvOrDefault("INTERLINK_TLS_ENABLED", "false") == trueValue + criConfig.TLS.CertFile = os.Getenv("INTERLINK_TLS_CERT_FILE") + criConfig.TLS.KeyFile = os.Getenv("INTERLINK_TLS_KEY_FILE") + criConfig.TLS.CACertFile = os.Getenv("INTERLINK_TLS_CA_FILE") + + // Set up logging + logger := logrus.StandardLogger() + logger.SetLevel(logrus.InfoLevel) + if criConfig.VerboseLogging { + logger.SetLevel(logrus.DebugLevel) + } else if criConfig.ErrorsOnlyLogging { + logger.SetLevel(logrus.ErrorLevel) + } + + log.L = logruslogger.FromLogrus(logrus.NewEntry(logger)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Set up tracing if enabled + if os.Getenv("ENABLE_TRACING") == "1" { + shutdown, err := interlink.InitTracer(ctx, "InterLink-CRI-") + if err != nil { + log.G(ctx).Fatal(err) + } + defer func() { + if err = shutdown(ctx); err != nil { + log.G(ctx).Fatal("failed to shutdown TracerProvider: %w", err) + } + }() + + log.G(ctx).Info("Tracer setup succeeded") + trace.T = opentelemetry.Adapter{} + } + + log.G(ctx).Info("Starting InterLink CRI version: ", CRIVersion) + log.G(ctx).Info("InterLink API server: ", criConfig.InterlinkURL, ":", criConfig.InterlinkPort) + log.G(ctx).Info("CRI socket: ", *socketPath) + + // Create and start the CRI runtime + interlinkRuntime := NewFakeRemoteRuntime(ctx, criConfig) + + // Handle shutdown gracefully + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + log.G(ctx).Info("Received shutdown signal, stopping CRI runtime...") + interlinkRuntime.Stop() + + // Clean up socket file + if addr, _, err := util.GetAddressAndDialer(*socketPath); err == nil { + if _, err := os.Stat(addr); err == nil { + os.Remove(addr) + log.G(ctx).Info("Cleaned up socket file: ", addr) + } + } + os.Exit(0) + }() + + // Start the CRI server + err := interlinkRuntime.Start(*socketPath) + if err != nil { + interlinkRuntime.Stop() + log.G(ctx).Fatal("Failed to start CRI server: ", err) + } + + // Keep the main goroutine alive + select {} +} + +func getEnvOrDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} diff --git a/cmd/interlink/main.go b/cmd/interlink/main.go index a373723a..2fa67bb2 100644 --- a/cmd/interlink/main.go +++ b/cmd/interlink/main.go @@ -22,8 +22,7 @@ import ( "github.com/interlink-hq/interlink/pkg/interlink" "github.com/interlink-hq/interlink/pkg/interlink/api" - "github.com/interlink-hq/interlink/pkg/virtualkubelet" - "k8s.io/cri-client/pkg/util" + vkconfig "github.com/interlink-hq/interlink/pkg/virtualkubelet" ) // UnixSocketRoundTripper is a custom RoundTripper for Unix socket connections @@ -89,7 +88,7 @@ func main() { flag.Parse() if *printVersion { - fmt.Println(virtualkubelet.KubeletVersion) + fmt.Println(vkconfig.KubeletVersion) return } var cancel context.CancelFunc @@ -130,7 +129,7 @@ func main() { log.G(ctx).Info(interLinkConfig) - log.G(ctx).Info("interLink version: ", virtualkubelet.KubeletVersion) + log.G(ctx).Info("interLink version: ", vkconfig.KubeletVersion) sidecarEndpoint := "" var socketPath string @@ -255,20 +254,4 @@ func main() { default: log.G(ctx).Fatal("Interlink URL should start with unix://, http://, or https://. Getting: ", interLinkConfig.InterlinkAddress) } - - interlinkRuntime := NewFakeRemoteRuntime() - err = interlinkRuntime.Start("unix:///tmp/kubelet_remote_1000.sock") - if err != nil { - interlinkRuntime.Stop() - log.G(ctx).Fatal(err) - } - defer func() { - interlinkRuntime.Stop() - // clear endpoint file - if addr, _, err := util.GetAddressAndDialer("unix:///tmp/kubelet_remote_1000.sock"); err == nil { - if _, err := os.Stat(addr); err == nil { - os.Remove(addr) - } - } - }() } diff --git a/cmd/virtual-kubelet/main.go b/cmd/virtual-kubelet/main.go index 2893e890..42aca146 100644 --- a/cmd/virtual-kubelet/main.go +++ b/cmd/virtual-kubelet/main.go @@ -212,9 +212,23 @@ func createCertPool(ctx context.Context, interLinkConfig commonIL.Config) *x509. } // createHTTPServer creates and starts the HTTPS server -func createHTTPServer(ctx context.Context, cfg Config, interLinkConfig commonIL.Config) *http.ServeMux { +func createHTTPServer(ctx context.Context, cfg Config, interLinkConfig commonIL.Config, kubeClient *kubernetes.Clientset) *http.ServeMux { mux := http.NewServeMux() - retriever := commonIL.NewSelfSignedCertificateRetriever(cfg.NodeName, net.ParseIP(cfg.InternalIP)) + + var retriever commonIL.Crtretriever + var err error + + // Choose certificate retriever based on configuration + if interLinkConfig.KubeletHTTPSCertMode == "csr" { + retriever, err = commonIL.NewCSRCertificateRetriever(kubeClient, cfg.NodeName, net.ParseIP(cfg.InternalIP)) + if err != nil { + log.G(ctx).Fatalf("Failed to create CSR certificate retriever: %v", err) + } + log.G(ctx).Info("Using Kubernetes CSR-based certificate retriever for virtual kubelet HTTPS server") + } else { + retriever = commonIL.NewSelfSignedCertificateRetriever(cfg.NodeName, net.ParseIP(cfg.InternalIP)) + log.G(ctx).Info("Using self-signed certificate retriever for virtual kubelet HTTPS server") + } kubeletURL, _ := getKubeletEndpoint() server := &http.Server{ @@ -392,11 +406,11 @@ func main() { DaemonPort: dport, } - mux := createHTTPServer(ctx, cfg, interLinkConfig) + kubecfg, localClient := setupKubernetesClient(ctx) - transport := createHTTPTransport(ctx, interLinkConfig, vkConfig) + mux := createHTTPServer(ctx, cfg, interLinkConfig, localClient) - kubecfg, localClient := setupKubernetesClient(ctx) + transport := createHTTPTransport(ctx, interLinkConfig, vkConfig) nodeProvider, err := commonIL.NewProvider( ctx, diff --git a/docs/docs/guides/07-mtls-deployment.mdx b/docs/docs/guides/07-mtls-deployment.mdx index 9326d229..1f9df754 100644 --- a/docs/docs/guides/07-mtls-deployment.mdx +++ b/docs/docs/guides/07-mtls-deployment.mdx @@ -21,7 +21,107 @@ Before setting up mTLS, ensure you have: ## Certificate Generation -### Generate Certificates for mTLS +You can generate certificates for mTLS using either Kubernetes-native CSR API or traditional OpenSSL commands. + +### Option A: Kubernetes CSR API (Recommended) + +This approach uses Kubernetes' built-in Certificate Signing Request API for a more cloud-native certificate management experience. + +#### Step 1: Generate Private Keys + +```bash +# Generate server private key +openssl genrsa -out server-key.pem 4096 + +# Generate client private key +openssl genrsa -out client-key.pem 4096 +``` + +#### Step 2: Create Certificate Signing Requests + +```bash +# Generate server CSR +openssl req -new -key server-key.pem -out server.csr -subj "/CN=system:node:interlink-server/O=system:nodes" + +# Generate client CSR +openssl req -new -key client-key.pem -out client.csr -subj "/CN=system:node:interlink-client/O=system:nodes" +``` + +#### Step 3: Create Kubernetes CSR Resources + +Create the server CSR manifest: + +```yaml title="server-csr.yaml" +apiVersion: certificates.k8s.io/v1 +kind: CertificateSigningRequest +metadata: + name: interlink-server-csr +spec: + request: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0K... # base64 encoded server.csr + signerName: kubernetes.io/kubelet-serving + expirationSeconds: 31536000 # 1 year + usages: + - digital signature + - key encipherment + - server auth +``` + +Create the client CSR manifest: + +```yaml title="client-csr.yaml" +apiVersion: certificates.k8s.io/v1 +kind: CertificateSigningRequest +metadata: + name: interlink-client-csr +spec: + request: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0K... # base64 encoded client.csr + signerName: kubernetes.io/kube-apiserver-client + expirationSeconds: 31536000 # 1 year + usages: + - digital signature + - key encipherment + - client auth +``` + +#### Step 4: Apply CSRs and Approve + +```bash +# Encode CSR files to base64 and update manifests +SERVER_CSR=$(cat server.csr | base64 | tr -d '\n') +CLIENT_CSR=$(cat client.csr | base64 | tr -d '\n') + +# Update the manifests with actual base64 encoded CSRs +sed -i "s/LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0K.../$SERVER_CSR/" server-csr.yaml +sed -i "s/LS0tLS1CRUdJTiBDRVJUSUZJQ0FURSBSRVFVRVNULS0tLS0K.../$CLIENT_CSR/" client-csr.yaml + +# Apply CSRs to Kubernetes +kubectl apply -f server-csr.yaml +kubectl apply -f client-csr.yaml + +# Approve CSRs (requires cluster-admin privileges) +kubectl certificate approve interlink-server-csr +kubectl certificate approve interlink-client-csr +``` + +#### Step 5: Extract Certificates + +```bash +# Extract server certificate +kubectl get csr interlink-server-csr -o jsonpath='{.status.certificate}' | base64 -d > server-cert.pem + +# Extract client certificate +kubectl get csr interlink-client-csr -o jsonpath='{.status.certificate}' | base64 -d > client-cert.pem + +# Get cluster CA certificate +kubectl config view --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 -d > ca.pem + +# Clean up CSR files +rm server.csr client.csr server-csr.yaml client-csr.yaml +``` + +### Option B: Manual OpenSSL (Alternative) + +If you prefer traditional certificate generation or don't have cluster-admin access for CSR approval: ```bash # Generate CA private key diff --git a/docs/docs/guides/09-virtual-kubelet-certificates.mdx b/docs/docs/guides/09-virtual-kubelet-certificates.mdx new file mode 100644 index 00000000..4d6d52da --- /dev/null +++ b/docs/docs/guides/09-virtual-kubelet-certificates.mdx @@ -0,0 +1,261 @@ +--- +sidebar_position: 9 +--- + +# Virtual Kubelet Certificate Management + +This guide covers certificate management options for InterLink's virtual-kubelet HTTPS server. The virtual-kubelet provides multiple ways to manage its server certificates, from automatic Kubernetes-native certificate management to simple self-signed certificates. + +## Overview + +The virtual-kubelet runs an HTTPS server that needs valid TLS certificates. InterLink supports two certificate management modes: + +1. **CSR-based certificates** - Uses Kubernetes Certificate Signing Request API (recommended) +2. **Self-signed certificates** - Automatically generated self-signed certificates (default) + +## CSR-Based Certificate Management (Recommended) + +### What is CSR-Based Certificate Management? + +CSR (Certificate Signing Request) based certificate management leverages Kubernetes' built-in certificate management infrastructure. The virtual-kubelet automatically: + +- Generates private keys +- Creates certificate signing requests +- Submits CSRs to the Kubernetes API server +- Manages certificate lifecycle and renewal + +### Benefits + +- **Kubernetes-native**: Integrates with cluster's certificate management +- **Automatic rotation**: Certificates are renewed automatically before expiration +- **Proper CA chain**: Certificates signed by the cluster's certificate authority +- **Standard compliance**: Follows the same patterns as real Kubernetes nodes +- **Better security**: No manual certificate management required + +### Configuration + +To enable CSR-based certificates, add the following to your virtual-kubelet configuration: + +```yaml title="VirtualKubeletConfig.yaml" +# Enable CSR-based certificate management +KubeletHTTPSCertMode: "csr" + +# Standard virtual-kubelet configuration +InterlinkURL: https://your-interlink-server +InterlinkPort: "3000" +Resources: + CPU: "8" + Memory: "16Gi" + Pods: "100" +``` + +### Requirements + +1. **RBAC Permissions**: The virtual-kubelet service account needs permissions to create and manage CSRs: + +```yaml title="csr-permissions.yaml" +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: virtual-kubelet-csr +rules: +- apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["create", "get", "list", "watch"] +- apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests/approval"] + verbs: ["update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: virtual-kubelet-csr +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtual-kubelet-csr +subjects: +- kind: ServiceAccount + name: virtual-kubelet + namespace: interlink +``` + +2. **Cluster Configuration**: The cluster must have the `kubernetes.io/kubelet-serving` signer available and configured. + +3. **CSR Approval**: Depending on your cluster configuration, CSRs may need manual approval: + +```bash +# Check pending CSRs +kubectl get csr + +# Approve virtual-kubelet CSRs +kubectl certificate approve +``` + +### Certificate Details + +When using CSR mode, certificates will have: +- **Subject**: `CN=system:node:, O=system:nodes` +- **SANs**: Node IP address, node name, and `.local` +- **Key Usage**: Digital Signature, Key Encipherment, Server Auth +- **Validity**: 1 year (automatically renewed) + +## Self-Signed Certificate Management (Default) + +### What are Self-Signed Certificates? + +Self-signed certificates are automatically generated by the virtual-kubelet without requiring any external certificate authority. This is the default mode for backward compatibility. + +### When to Use Self-Signed Certificates + +- Quick development and testing setups +- Environments where CSR approval workflow is not desired +- Clusters without proper certificate signing infrastructure +- When you need minimal configuration + +### Configuration + +Self-signed mode is the default. You can explicitly configure it: + +```yaml title="VirtualKubeletConfig.yaml" +# Explicit self-signed mode (optional - this is the default) +KubeletHTTPSCertMode: "self-signed" + +# Or simply omit the KubeletHTTPSCertMode field entirely +InterlinkURL: https://your-interlink-server +InterlinkPort: "3000" +``` + +### Certificate Details + +Self-signed certificates have: +- **Subject**: `CN=system:node:, O=intertwin.eu` +- **SANs**: Node IP address +- **Key Usage**: Digital Signature, Client Auth, Server Auth +- **Validity**: 1 year (automatically renewed when near expiration) +- **Algorithm**: Ed25519 + +## Helm Chart Configuration + +### CSR Mode + +```yaml title="values.yaml" +# Enable CSR-based certificates +virtualKubelet: + config: + KubeletHTTPSCertMode: "csr" + +# Ensure proper RBAC permissions +rbac: + create: true + extraRules: + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["create", "get", "list", "watch"] + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests/approval"] + verbs: ["update"] +``` + +### Self-Signed Mode + +```yaml title="values.yaml" +# Use self-signed certificates (default) +virtualKubelet: + config: + KubeletHTTPSCertMode: "self-signed" + # or omit this field entirely +``` + +## Troubleshooting + +### CSR Mode Issues + +1. **CSR Creation Failures** + ```bash + # Check virtual-kubelet logs + kubectl logs -n interlink deployment/virtual-kubelet + + # Check RBAC permissions + kubectl auth can-i create certificatesigningrequests --as=system:serviceaccount:interlink:virtual-kubelet + ``` + +2. **Pending CSRs** + ```bash + # List pending CSRs + kubectl get csr | grep Pending + + # Approve specific CSR + kubectl certificate approve virtual-kubelet-csr- + ``` + +3. **Certificate Renewal Issues** + ```bash + # Check certificate expiration + kubectl get csr | grep virtual-kubelet + + # Delete old CSRs to trigger renewal + kubectl delete csr + ``` + +### Self-Signed Mode Issues + +1. **Certificate Trust Issues** + - Self-signed certificates will show as untrusted in browsers + - Use `--insecure` flag or proper CA configuration for clients + +2. **Certificate Regeneration** + - Certificates are automatically regenerated before expiration + - To force regeneration, restart the virtual-kubelet pod + +### General Debugging + +```bash +# Check virtual-kubelet server certificate +openssl s_client -connect :10250 -servername + +# Verify certificate details +echo | openssl s_client -connect :10250 2>/dev/null | openssl x509 -text -noout +``` + +## Migration Between Modes + +### From Self-Signed to CSR + +1. Update configuration to use CSR mode +2. Ensure RBAC permissions are configured +3. Restart virtual-kubelet +4. Approve any pending CSRs if needed + +### From CSR to Self-Signed + +1. Update configuration to use self-signed mode +2. Restart virtual-kubelet +3. Clean up old CSRs (optional): + ```bash + kubectl delete csr -l app=virtual-kubelet + ``` + +## Security Considerations + +### CSR Mode Security + +- CSRs are automatically approved by the certificate manager +- Certificates are stored in `/tmp/certs` inside the container +- Private keys are generated using secure random sources +- Regular certificate rotation minimizes exposure window + +### Self-Signed Mode Security + +- Certificates are not verified by external CA +- Suitable for internal/development use +- Consider using CSR mode for production deployments +- Private keys are ephemeral and container-local + +## Best Practices + +1. **Use CSR mode for production** - Better security and integration +2. **Monitor certificate expiration** - Set up alerts for certificate renewal +3. **Configure proper RBAC** - Limit CSR permissions to necessary scope +4. **Regular updates** - Keep virtual-kubelet updated for security patches +5. **Backup considerations** - Certificates are automatically managed, no backup needed \ No newline at end of file diff --git a/docs/docs/other-integrations/_category_.json b/docs/docs/other-integrations/_category_.json new file mode 100644 index 00000000..558c498f --- /dev/null +++ b/docs/docs/other-integrations/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Other Integrations", + "position": 4, + "link": { + "type": "generated-index", + "description": "Alternative ways to integrate interLink with Kubernetes and container runtimes." + } +} \ No newline at end of file diff --git a/docs/docs/other-integrations/cri/00-overview.md b/docs/docs/other-integrations/cri/00-overview.md new file mode 100644 index 00000000..54ae2f49 --- /dev/null +++ b/docs/docs/other-integrations/cri/00-overview.md @@ -0,0 +1,116 @@ +# CRI Integration Overview + +The interLink Container Runtime Interface (CRI) implementation enables direct integration with kubelet, allowing interLink to function as a native Kubernetes container runtime. This provides an alternative to the Virtual Kubelet approach for running containers on remote resources. + +## What is CRI Integration? + +The Container Runtime Interface (CRI) is a plugin interface that enables kubelet to use different container runtimes. interLink's CRI implementation acts as a bridge between kubelet and interLink's remote execution capabilities. + +```mermaid +graph LR + A[Kubelet] -->|CRI gRPC| B[InterLink CRI] + B -->|HTTP REST| C[InterLink API] + C -->|Plugin Interface| D[Remote Resources] + D --> E[SLURM Cluster] + D --> F[Docker Nodes] + D --> G[Cloud Providers] +``` + +## Benefits of CRI Integration + +### Standard Kubernetes Experience +- **Native kubectl support** - All standard kubectl commands work unchanged +- **Kubernetes API compatibility** - Full integration with Kubernetes controllers +- **Resource management** - Leverage Kubernetes native resource scheduling +- **Service discovery** - Standard Kubernetes service networking + +### Simplified Architecture +- **Direct kubelet integration** - No need for Virtual Kubelet deployment +- **Reduced complexity** - Fewer components to manage and monitor +- **Standard workflows** - Use existing Kubernetes deployment patterns + +### Enhanced Features +- **Pod lifecycle management** - Complete container lifecycle support +- **Resource quotas** - Standard Kubernetes resource management +- **Health checks** - Native readiness and liveness probe support +- **Log streaming** - Standard kubectl logs functionality + +## When to Use CRI vs Virtual Kubelet + +### Use CRI Integration When: +- You want direct kubelet integration +- You prefer standard Kubernetes workflows +- You need tight integration with Kubernetes scheduling +- You want to minimize architectural complexity + +### Use Virtual Kubelet When: +- You need to represent remote resources as virtual nodes +- You want more flexibility in node characteristics +- You need custom node labeling and tainting +- You're running in managed Kubernetes environments + +## Architecture Comparison + +### Virtual Kubelet Approach +``` +┌─────────────┐ ┌──────────────┐ ┌─────────────┐ +│ kube-apiserver │ │ Virtual Kubelet │ │ InterLink │ +│ │────┤ │────┤ Plugin │ +└─────────────┘ └──────────────┘ └─────────────┘ +``` + +### CRI Integration Approach +``` +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ Kubelet │ │ InterLink │ │ InterLink │ +│ │────┤ CRI Runtime │────┤ Plugin │ +└─────────────┘ └─────────────┘ └─────────────┘ +``` + +## Getting Started + +1. **[Quick Start](01-quickstart.md)** - Get up and running in 5 minutes +2. **[Usage Guide](02-usage-guide.md)** - Comprehensive setup and configuration +3. **[Technical Reference](03-technical-reference.md)** - Implementation details and architecture + +## Supported Features + +### ✅ Fully Supported +- Container creation, start, stop, removal +- Pod sandbox management +- Resource limits and requests +- Environment variables +- Basic networking +- Log retrieval +- Command execution + +### 🚧 Partial Support +- Volume mounts (basic support) +- Init containers +- Security contexts +- Network policies + +### 📋 Planned Features +- Advanced volume types +- Pod security policies +- Custom resource definitions +- Enhanced networking + +## Requirements + +- **Kubernetes 1.24+** with CRI support +- **Go 1.21+** for building +- **InterLink 0.6+** with CRI implementation +- **Linux-based kubelet** (recommended) + +## Community and Support + +The CRI integration is actively developed and maintained. For questions, issues, or contributions: + +- **GitHub Issues**: [Report bugs or request features](https://github.com/interlink-hq/interlink/issues) +- **Discussions**: [Community discussions](https://github.com/interlink-hq/interlink/discussions) +- **Documentation**: [Main interLink documentation](https://interlink.readthedocs.io/) + +--- + +Ready to get started? Check out the **[Quick Start Guide](01-quickstart.md)** for a fast setup, or dive into the **[Usage Guide](02-usage-guide.md)** for comprehensive configuration options. \ No newline at end of file diff --git a/docs/docs/other-integrations/cri/01-quickstart.md b/docs/docs/other-integrations/cri/01-quickstart.md new file mode 100644 index 00000000..c3cf8f2d --- /dev/null +++ b/docs/docs/other-integrations/cri/01-quickstart.md @@ -0,0 +1,282 @@ +# InterLink CRI Quick Start Guide + +This guide provides a quick setup and usage reference for the interLink CRI implementation. + +## Quick Setup (5 minutes) + +### 1. Build and Start InterLink CRI + +```bash +# Build +make interlink + +# Start with default configuration +./bin/interlink & + +# CRI socket will be available at: unix:///tmp/kubelet_remote_1000.sock +``` + +### 2. Configure Kubelet + +```bash +# Minimal kubelet configuration +kubelet --container-runtime=remote \ + --container-runtime-endpoint=unix:///tmp/kubelet_remote_1000.sock \ + --image-service-endpoint=unix:///tmp/kubelet_remote_1000.sock \ + --kubeconfig=/etc/kubernetes/kubelet.conf +``` + +### 3. Test with Simple Pod + +```yaml +# save as test-pod.yaml +apiVersion: v1 +kind: Pod +metadata: + name: hello-interlink +spec: + containers: + - name: hello + image: hello-world +``` + +```bash +kubectl apply -f test-pod.yaml +kubectl get pods -w +``` + +## Common Commands + +### CRI Operations + +```bash +# List containers +crictl --runtime-endpoint unix:///tmp/kubelet_remote_1000.sock ps + +# List pod sandboxes +crictl --runtime-endpoint unix:///tmp/kubelet_remote_1000.sock pods + +# Get container logs +crictl --runtime-endpoint unix:///tmp/kubelet_remote_1000.sock logs + +# Execute in container +crictl --runtime-endpoint unix:///tmp/kubelet_remote_1000.sock exec -it /bin/sh +``` + +### InterLink Status + +```bash +# Check interLink health +curl -X POST http://localhost:8080/pinglink + +# Get pod statuses +curl -X GET http://localhost:8080/status + +# Check interLink logs +journalctl -u interlink -f +``` + +### Kubernetes Operations + +```bash +# Standard kubectl commands work normally +kubectl get pods +kubectl describe pod +kubectl logs +kubectl exec -it -- /bin/bash +kubectl delete pod +``` + +## Configuration Templates + +### Minimal InterLinkConfig.yaml + +```yaml +InterlinkAddress: "http://localhost" +Interlinkport: "8080" +SidecarURL: "http://localhost" +Sidecarport: "4000" +VerboseLogging: true +``` + +### Production Kubelet Config + +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +containerRuntimeEndpoint: "unix:///tmp/kubelet_remote_1000.sock" +imageServiceEndpoint: "unix:///tmp/kubelet_remote_1000.sock" +containerRuntime: "remote" +runtimeRequestTimeout: "15m" +clusterDomain: "cluster.local" +clusterDNS: ["10.96.0.10"] +authentication: + webhook: + enabled: true +authorization: + mode: Webhook +``` + +## Troubleshooting Quick Fixes + +### Issue: CRI socket not found +```bash +# Check if interLink is running +ps aux | grep interlink + +# Check socket exists +ls -la /tmp/kubelet_remote_1000.sock + +# Restart interLink +pkill interlink +./bin/interlink & +``` + +### Issue: Pod stuck in Pending +```bash +# Check container creation +crictl --runtime-endpoint unix:///tmp/kubelet_remote_1000.sock ps -a + +# Check interLink plugin connectivity +curl -X POST http://localhost:4000/status + +# Check kubelet logs +journalctl -u kubelet -f +``` + +### Issue: Container logs not available +```bash +# Direct CRI log access +crictl --runtime-endpoint unix:///tmp/kubelet_remote_1000.sock logs + +# Check interLink log endpoint +curl -X GET http://localhost:8080/getLogs +``` + +## Example Workloads + +### Batch Job + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: pi-calculation +spec: + template: + spec: + containers: + - name: pi + image: perl:5.32 + command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"] + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "256Mi" + restartPolicy: Never + backoffLimit: 4 +``` + +### Web Service + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx-server + labels: + app: nginx +spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 + env: + - name: NGINX_HOST + value: "localhost" + resources: + requests: + cpu: "100m" + memory: "128Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-service +spec: + selector: + app: nginx + ports: + - port: 80 + targetPort: 80 + type: ClusterIP +``` + +### Init Container Example + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: init-demo +spec: + initContainers: + - name: init-setup + image: busybox:1.35 + command: ['sh', '-c', 'echo "Setup complete" > /shared/status'] + volumeMounts: + - name: shared-data + mountPath: /shared + containers: + - name: main-app + image: busybox:1.35 + command: ['sh', '-c', 'cat /shared/status && sleep 3600'] + volumeMounts: + - name: shared-data + mountPath: /shared + volumes: + - name: shared-data + emptyDir: {} +``` + +## Performance Tips + +1. **Resource Requests**: Always set CPU/memory requests for better scheduling +2. **Image Pulls**: Use image pull policies appropriate for your environment +3. **Health Checks**: Implement proper readiness and liveness probes +4. **Graceful Shutdown**: Set appropriate termination grace periods + +## Environment Variables + +Key environment variables for the CRI implementation: + +```bash +# Enable debug logging +export INTERLINK_VERBOSE=true + +# Custom CRI socket path +export CRI_SOCKET_PATH=/var/run/interlink/cri.sock + +# Plugin timeout +export PLUGIN_TIMEOUT=300s + +# OpenTelemetry tracing +export ENABLE_TRACING=1 +export OTEL_SERVICE_NAME=interlink-cri +``` + +## Next Steps + +After getting the basic setup working: + +1. **Configure your plugin**: Set up SLURM, Docker, or custom plugin +2. **Security**: Enable TLS and proper authentication +3. **Monitoring**: Set up metrics and observability +4. **Production**: Configure resource quotas and limits +5. **Scale**: Deploy across multiple nodes + +For detailed information, see the full [CRI Usage Guide](02-usage-guide.md). \ No newline at end of file diff --git a/docs/docs/other-integrations/cri/02-usage-guide.md b/docs/docs/other-integrations/cri/02-usage-guide.md new file mode 100644 index 00000000..eb9b377a --- /dev/null +++ b/docs/docs/other-integrations/cri/02-usage-guide.md @@ -0,0 +1,673 @@ +# InterLink CRI Implementation Usage Guide + +This document provides comprehensive guidance on using the interLink CRI (Container Runtime Interface) implementation as a kubelet container runtime. + +## Overview + +The interLink CRI implementation bridges the Kubernetes CRI with interLink's remote execution capabilities, allowing kubelet to manage containers through interLink's plugin architecture. This enables running containers on remote resources (HPC clusters, cloud providers, etc.) while maintaining full CRI compatibility. + +## Architecture + +The interLink CRI implementation provides a standalone binary that acts as a bridge between kubelet and the interLink API server: + +``` +┌─────────────┐ CRI gRPC ┌─────────────────┐ HTTP/REST ┌─────────────────┐ Plugin API ┌─────────────┐ +│ Kubelet │ ─────────────▶│ InterLink CRI │ ───────────────▶│ InterLink API │ ──────────────▶│ InterLink │ +│ │ (socket) │ Binary │ (AuthN/AuthZ) │ Server │ │ Plugin │ +└─────────────┘ └─────────────────┘ └─────────────────┘ └─────────────┘ + │ │ + ▼ ▼ + ┌─────────────────┐ ┌─────────────────┐ + │ Pod Status │ │ Remote Resource │ + │ Tracking │ │ (HPC/Cloud) │ + └─────────────────┘ └─────────────────┘ +``` + +**Key Benefits of Standalone Architecture:** +- Clean separation between CRI and interLink API server +- Independent deployment and scaling +- Standard CRI binary pattern (like containerd, cri-o) +- Flexible authentication/authorization options +- Can be used alongside existing container runtimes + +## Prerequisites + +1. **InterLink Server**: A running interLink API server with plugin configured +2. **Go 1.21+**: For building the CRI implementation +3. **Kubernetes Cluster**: With kubelet configuration access +4. **Network Connectivity**: Between kubelet and interLink CRI socket + +## Installation and Setup + +### 1. Build InterLink CRI Binary + +Build the standalone CRI binary: + +```bash +# Clone the interLink repository +git clone https://github.com/interlink-hq/interlink.git +cd interlink + +# Build standalone CRI binary +go build -o bin/interlink-cri cmd/interlink-cri/*.go + +# Or use make target +make interlink-cri +``` + +The CRI binary is completely independent from the interLink API server and can be deployed separately. + +### 2. Configure InterLink CRI Binary + +Create a CRI-specific configuration file (`InterLinkCRIConfig.yaml`): + +```yaml +# InterLink API Server connection +InterlinkAddress: "https://interlink-api.example.com" # InterLink API server endpoint +Interlinkport: "8080" + +# CRI Socket configuration +CRI: + SocketPath: "unix:///var/run/interlink/cri.sock" + RuntimeHandler: "interlink" + +# Logging configuration +VerboseLogging: true +ErrorsOnlyLogging: false + +# TLS Configuration for mTLS authentication with InterLink API server +TLS: + Enabled: true + CertFile: "/etc/interlink/cri/tls/client.crt" # Client certificate for mTLS + KeyFile: "/etc/interlink/cri/tls/client.key" # Client private key + CACertFile: "/etc/interlink/cri/tls/ca.crt" # CA certificate for server verification + +# Local data for pod tracking +DataRootFolder: "/var/lib/interlink-cri" +``` + +**Note**: This configuration is for the CRI binary to connect to an existing interLink API server. The API server itself has its own separate configuration. + +### 3. Configure Authentication + +The CRI implementation supports OAuth token authentication and mTLS: + +#### OAuth Token Authentication + +For OAuth token authentication, the CRI implementation uses OIDC tokens following the same pattern as virtual-kubelet. Set the token file path via environment variable: + +```bash +export VK_TOKEN_FILE="/etc/interlink/tokens/oidc.token" +``` + +Refer to the Virtual Kubelet OAuth configuration guide for complete OIDC token setup instructions. + +**Note**: OIDC tokens are only required when using OAuth authentication. If using mTLS certificate authentication instead, token setup can be skipped. + +#### mTLS Certificate Authentication + +Generate client certificates for mutual TLS authentication: + +```bash +# Generate client certificate (example with openssl) +openssl genrsa -out client.key 2048 +openssl req -new -key client.key -out client.csr -subj "/CN=interlink-cri" +openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 365 +``` + +### 4. Start InterLink CRI Binary + +```bash +# Start the standalone CRI binary +./bin/interlink-cri --config InterLinkCRIConfig.yaml + +# Or run as a systemd service +sudo systemctl start interlink-cri +``` + +The CRI server will be available at the configured socket path (default: `unix:///var/run/interlink/cri.sock`). + +#### Systemd Service Configuration + +Create a systemd service file (`/etc/systemd/system/interlink-cri.service`): + +```ini +[Unit] +Description=InterLink CRI Runtime +Documentation=https://interlink.readthedocs.io/ +After=network.target +Wants=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/interlink-cri --config /etc/interlink/cri/config.yaml +Restart=always +RestartSec=5 +KillMode=mixed +User=root +Group=root + +# Resource limits +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity + +[Install] +WantedBy=multi-user.target +``` + +Enable and start the service: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable interlink-cri +sudo systemctl start interlink-cri +``` + +## Kubelet Configuration + +### 1. Configure Kubelet for Remote CRI (Primary Runtime) + +For using interLink CRI as the primary container runtime, create or modify the kubelet configuration file (`/etc/kubernetes/kubelet/config.yaml`): + +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration + +# CRI Configuration - points to InterLink CRI binary socket +containerRuntimeEndpoint: "unix:///var/run/interlink/cri.sock" +imageServiceEndpoint: "unix:///var/run/interlink/cri.sock" + +# Runtime configuration +containerRuntime: "remote" +runtimeRequestTimeout: "15m" + +# Node configuration +staticPodPath: "/etc/kubernetes/manifests" +clusterDomain: "cluster.local" +clusterDNS: + - "10.96.0.10" + +# Authentication +authentication: + anonymous: + enabled: false + webhook: + enabled: true + +# Authorization +authorization: + mode: Webhook + +# Feature gates (if needed) +featureGates: + CRIContainerLogRotation: true +``` + +### 2. Configure Multiple Container Runtimes (RuntimeClass) + +For scenarios where interLink CRI is an additional runtime alongside the default container runtime (e.g., containerd), configure RuntimeClass: + +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration + +# Primary CRI (e.g., containerd) +containerRuntimeEndpoint: "unix:///run/containerd/containerd.sock" +imageServiceEndpoint: "unix:///run/containerd/containerd.sock" + +# Runtime configuration +containerRuntime: "remote" +runtimeRequestTimeout: "15m" + +# Enable RuntimeClass feature +featureGates: + RuntimeClass: true +``` + +Create a RuntimeClass for interLink: + +```yaml +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: interlink +handler: interlink +overhead: + podFixed: + memory: "128Mi" + cpu: "100m" +scheduling: + nodeClassification: + tolerations: + - key: "interlink.io/no-schedule" + operator: "Exists" + effect: "NoSchedule" +``` + +Configure the interLink CRI to register as a runtime handler: + +```yaml +# In InterLinkConfig.yaml +CRI: + Enabled: true + SocketPath: "unix:///var/run/interlink/interlink.sock" + RuntimeHandler: "interlink" +``` + +### 3. Start Kubelet with CRI Configuration + +```bash +# Start kubelet with the configuration +kubelet --config=/etc/kubernetes/kubelet/config.yaml \ + --kubeconfig=/etc/kubernetes/kubelet/kubeconfig \ + --container-runtime-endpoint=unix:///var/run/interlink/cri.sock \ + --v=2 +``` + +## Usage Examples + +### 1. Deploy a Simple Pod (Primary Runtime) + +Create a test pod for primary runtime (`test-pod.yaml`): + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: test-pod + namespace: default +spec: + containers: + - name: test-container + image: nginx:latest + ports: + - containerPort: 80 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + env: + - name: ENV_VAR + value: "test-value" +``` + +### 2. Deploy a Pod with Specific RuntimeClass + +For multi-runtime setups, specify the RuntimeClass to use interLink CRI (`test-pod-interlink.yaml`): + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-interlink + namespace: default +spec: + runtimeClassName: interlink # Use interLink CRI runtime + containers: + - name: hpc-workload + image: tensorflow/tensorflow:latest-gpu + command: ["python3", "/app/train.py"] + resources: + requests: + memory: "2Gi" + cpu: "4" + nvidia.com/gpu: "1" + limits: + memory: "8Gi" + cpu: "8" + nvidia.com/gpu: "2" + env: + - name: CUDA_VISIBLE_DEVICES + value: "0,1" + volumeMounts: + - name: data-volume + mountPath: /data + volumes: + - name: data-volume + persistentVolumeClaim: + claimName: hpc-data-pvc + tolerations: + - key: "interlink.io/no-schedule" + operator: "Exists" + effect: "NoSchedule" +``` + +### 3. Deploy Standard Pod (Default Runtime) + +Regular pods will use the default runtime (e.g., containerd) (`test-pod-default.yaml`): + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-default + namespace: default +spec: + # No runtimeClassName specified - uses default runtime + containers: + - name: web-server + image: nginx:latest + ports: + - containerPort: 80 + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" +``` + +Deploy the pods: + +```bash +# Deploy primary runtime pod +kubectl apply -f test-pod.yaml + +# Deploy interLink runtime pod (multi-runtime scenario) +kubectl apply -f test-pod-interlink.yaml + +# Deploy default runtime pod (multi-runtime scenario) +kubectl apply -f test-pod-default.yaml +``` + +### 4. Monitor Pod Status + +```bash +# Check pod status +kubectl get pods test-pod -o wide + +# Get detailed pod information +kubectl describe pod test-pod + +# Check container logs +kubectl logs test-pod +``` + +### 5. Interactive Container Access + +```bash +# Execute commands in the container +kubectl exec -it test-pod -- /bin/bash + +# Run specific commands +kubectl exec test-pod -- nginx -v +``` + +## Configuration Options + +### CRI Binary Configuration + +The standalone CRI binary configuration file supports the following options: + +```yaml +# InterLink API Server connection +InterlinkAddress: "https://interlink-api.example.com" +Interlinkport: "8080" + +# CRI Socket configuration +CRI: + SocketPath: "unix:///var/run/interlink/cri.sock" # Configurable socket path + RuntimeHandler: "interlink" # Runtime handler name + RequestTimeout: "15m" # Timeout for requests + +# Authentication +TLS: + Enabled: true + CertFile: "/etc/interlink/cri/tls/client.crt" + KeyFile: "/etc/interlink/cri/tls/client.key" + CACertFile: "/etc/interlink/cri/tls/ca.crt" + +# Logging +VerboseLogging: true +ErrorsOnlyLogging: false + +# Local storage for pod tracking +DataRootFolder: "/var/lib/interlink-cri" +``` + +### Environment Variables + +The CRI binary supports these environment variables: + +```bash +# OAuth token file (alternative to mTLS) +export VK_TOKEN_FILE="/etc/interlink/tokens/oidc.token" + +# Override config file location +export INTERLINK_CRI_CONFIG="/etc/interlink/cri/config.yaml" + +# Enable debug logging +export INTERLINK_CRI_DEBUG="true" +``` + +## Troubleshooting + +### Common Issues + +#### 1. CRI Socket Connection Failed + +**Error**: `Failed to connect to CRI socket` + +**Solution**: +```bash +# Check if interLink CRI binary is running +ps aux | grep interlink-cri +systemctl status interlink-cri + +# Check socket permissions and existence +ls -la /var/run/interlink/cri.sock + +# Verify socket is listening +ss -xlp | grep cri.sock + +# Check CRI binary logs +journalctl -u interlink-cri -f +``` + +#### 2. Container Creation Failed + +**Error**: `Failed to create container` + +**Check**: +1. InterLink API server connectivity +2. Authentication (OIDC token or mTLS) +3. Plugin connectivity from API server +4. Container resource requirements + +```bash +# Check CRI binary logs +journalctl -u interlink-cri -f + +# Check InterLink API server connectivity +curl -k https://interlink-api.example.com:8080/pinglink + +# Test authentication +curl -k -H "Authorization: Bearer $(cat /etc/interlink/tokens/oidc.token)" \ + https://interlink-api.example.com:8080/status + +# Check API server logs (if accessible) +journalctl -u interlink-api -f +``` + +#### 3. Pod Status Not Updated + +**Error**: Pods stuck in `Pending` state + +**Debug**: +```bash +# Check CRI container status through CRI binary +crictl --runtime-endpoint unix:///var/run/interlink/cri.sock ps + +# Check InterLink API server pod tracking +curl -k https://interlink-api.example.com:8080/status + +# Check CRI binary internal tracking +journalctl -u interlink-cri --since "5 minutes ago" +``` + +### Debugging Commands + +```bash +# List CRI containers via standalone CRI binary +crictl --runtime-endpoint unix:///var/run/interlink/cri.sock ps -a + +# Inspect container +crictl --runtime-endpoint unix:///var/run/interlink/cri.sock inspect + +# Check container logs +crictl --runtime-endpoint unix:///var/run/interlink/cri.sock logs + +# List pod sandboxes +crictl --runtime-endpoint unix:///var/run/interlink/cri.sock pods + +# Test CRI binary directly +curl -X POST --unix-socket /var/run/interlink/cri.sock \ + -H "Content-Type: application/json" \ + http://localhost/healthz +``` + +## Monitoring and Observability + +### Metrics + +The CRI implementation provides integration with interLink's observability: + +- **Container Lifecycle Metrics**: Creation, start, stop, removal +- **Pod Status Tracking**: State transitions and health +- **Request Tracing**: End-to-end request tracking through OpenTelemetry + +### Logging + +Enable detailed logging by setting: + +```yaml +VerboseLogging: true +``` + +Log locations: +- **InterLink CRI**: Container lifecycle events +- **InterLink API**: HTTP requests and responses +- **Plugin**: Remote execution logs + +## Advanced Configuration + +### Custom Resource Mapping + +Customize how CRI resource requirements map to interLink pods: + +```go +// In convertToInterLinkPod function +if containerConfig.Linux != nil && containerConfig.Linux.Resources != nil { + resources := v1.ResourceRequirements{} + + // Custom CPU mapping + if containerConfig.Linux.Resources.CpuQuota > 0 { + cpuLimit := containerConfig.Linux.Resources.CpuQuota / 1000 + resources.Limits = v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(cpuLimit, resource.DecimalSI), + } + } + + // Custom memory mapping + if containerConfig.Linux.Resources.MemoryLimitInBytes > 0 { + if resources.Limits == nil { + resources.Limits = v1.ResourceList{} + } + resources.Limits[v1.ResourceMemory] = *resource.NewQuantity( + containerConfig.Linux.Resources.MemoryLimitInBytes, + resource.BinarySI, + ) + } + + pod.Spec.Containers[0].Resources = resources +} +``` + +### Plugin Integration + +Configure specific plugin behavior through environment variables or configuration: + +```yaml +# Plugin-specific environment variables +PluginConfig: + SLURM_PARTITION: "gpu" + SLURM_QOS: "high" + DOCKER_REGISTRY: "registry.example.com" +``` + +## Best Practices + +### 1. Resource Management + +- Set appropriate resource requests and limits +- Monitor resource usage through interLink metrics +- Use resource quotas at namespace level + +### 2. Network Configuration + +- Ensure proper network policies for container communication +- Configure DNS resolution for service discovery +- Use appropriate service mesh integration if needed + +### 3. Security + +- Enable TLS for interLink communication +- Use proper RBAC for kubelet service account +- Secure the CRI socket with appropriate permissions + +### 4. Monitoring + +- Set up monitoring for both kubelet and interLink +- Monitor container lifecycle metrics +- Track resource utilization on remote resources + +## Migration Guide + +### From Docker to InterLink CRI + +1. **Backup Current Configuration**: + ```bash + cp /etc/kubernetes/kubelet/config.yaml /etc/kubernetes/kubelet/config.yaml.bak + ``` + +2. **Update Kubelet Configuration**: + ```yaml + # Change from Docker to InterLink CRI + containerRuntimeEndpoint: "unix:///tmp/kubelet_remote_1000.sock" + ``` + +3. **Restart Services**: + ```bash + systemctl restart kubelet + systemctl restart interlink + ``` + +4. **Verify Migration**: + ```bash + kubectl get nodes + kubectl get pods --all-namespaces + ``` + +## Support and Community + +- **Documentation**: [InterLink Documentation](https://interlink.readthedocs.io/) +- **Issues**: [GitHub Issues](https://github.com/interlink-hq/interlink/issues) +- **Discussions**: [GitHub Discussions](https://github.com/interlink-hq/interlink/discussions) +- **Slack**: [CNCF Slack #interlink](https://cloud-native.slack.com/channels/interlink) + +## Contributing + +To contribute to the CRI implementation: + +1. Fork the repository +2. Create a feature branch +3. Make your changes following the existing patterns +4. Add tests for new functionality +5. Submit a pull request + +For detailed contribution guidelines, see the project's contribution documentation. \ No newline at end of file diff --git a/docs/docs/other-integrations/cri/03-technical-reference.md b/docs/docs/other-integrations/cri/03-technical-reference.md new file mode 100644 index 00000000..0ece8d7a --- /dev/null +++ b/docs/docs/other-integrations/cri/03-technical-reference.md @@ -0,0 +1,539 @@ +# InterLink CRI Technical Reference + +This document provides technical details about the interLink CRI implementation, architecture decisions, and implementation patterns. + +## Architecture Overview + +### Component Interaction + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Kubelet │ │ InterLink CRI │ │ InterLink API │ +│ │ │ Implementation │ │ Server │ +├─────────────────┤ gRPC ├─────────────────┤ HTTP ├─────────────────┤ +│ CRI Client │◄────────┤ gRPC Server │────────►│ REST Handlers │ +│ (Container │ │ (RemoteRuntime) │ │ - Create │ +│ Management) │ │ │ │ - Delete │ +└─────────────────┘ └─────────────────┘ │ - Status │ + │ │ - Logs │ + ▼ └─────────────────┘ + ┌─────────────────┐ │ + │ Pod Status │ ▼ + │ Tracking │ ┌─────────────────┐ + │ (PodStatuses) │ │ Plugin System │ + └─────────────────┘ │ (SLURM, Docker, │ + │ Custom, etc.) │ + └─────────────────┘ +``` + +### Key Components + +#### 1. RemoteRuntime Structure + +```go +type RemoteRuntime struct { + server *grpc.Server // gRPC server for CRI + RuntimeService *apitest.FakeRuntimeService // CRI compatibility layer + ImageService *apitest.FakeImageService // Image management + InterLinkHandler *api.InterLinkHandler // Integration with interLink +} +``` + +#### 2. Container Lifecycle Integration + +The CRI implementation integrates with interLink's lifecycle through these key methods: + +- **CreateContainer**: Converts CRI → interLink pod format +- **StartContainer**: Updates interLink pod status tracking +- **StopContainer**: Manages container termination +- **RemoveContainer**: Cleans up tracking state + +## Implementation Details + +### Container Creation Flow + +```mermaid +sequenceDiagram + participant K as Kubelet + participant C as CRI Implementation + participant F as FakeRuntimeService + participant I as InterLink API + participant P as Plugin + + K->>C: CreateContainer(config) + C->>C: convertToInterLinkPod() + C->>F: CreateContainer() + C->>I: Store pod status + C->>K: ContainerID + + K->>C: StartContainer(containerID) + C->>F: StartContainer() + C->>C: updatePodStatusContainer() + Note over C: Update to Running state + C->>K: Success +``` + +### Pod Status Management + +The implementation maintains pod status through the `PodStatuses` global structure: + +```go +type MutexStatuses struct { + mu sync.Mutex + Statuses map[string]types.PodStatus +} + +var PodStatuses MutexStatuses +``` + +#### Helper Functions + +```go +// Safe access to pod status without exposing mutex +func getPodStatus(uid string) (types.PodStatus, bool) +func updatePodStatusContainer(sandboxID string, containerStatus v1.ContainerStatus) +func removePodStatusContainer(sandboxID string, containerName string) +func removePodStatus(uid string) +``` + +### Type Conversion + +#### CRI to Kubernetes Pod Conversion + +```go +func convertToInterLinkPod( + ctx context.Context, + sandboxConfig *kubeapi.PodSandboxConfig, + containerConfig *kubeapi.ContainerConfig +) *v1.Pod { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: sandboxConfig.Metadata.Name, + Namespace: sandboxConfig.Metadata.Namespace, + UID: ktypes.UID(sandboxConfig.Metadata.Uid), + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: containerConfig.Metadata.Name, + Image: containerConfig.Image.Image, + Command: containerConfig.Command, + Args: containerConfig.Args, + WorkingDir: containerConfig.WorkingDir, + }, + }, + }, + Status: v1.PodStatus{ + Phase: v1.PodPending, + }, + } + + // Environment variable conversion + if len(containerConfig.Envs) > 0 { + envVars := make([]v1.EnvVar, len(containerConfig.Envs)) + for i, env := range containerConfig.Envs { + envVars[i] = v1.EnvVar{ + Name: env.Key, + Value: env.Value, + } + } + pod.Spec.Containers[0].Env = envVars + } + + // Resource conversion + if containerConfig.Linux != nil && containerConfig.Linux.Resources != nil { + resources := v1.ResourceRequirements{} + + // CPU limits + if containerConfig.Linux.Resources.CpuQuota > 0 { + resources.Limits = v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity( + containerConfig.Linux.Resources.CpuQuota/1000, + resource.DecimalSI, + ), + } + } + + // Memory limits + if containerConfig.Linux.Resources.MemoryLimitInBytes > 0 { + if resources.Limits == nil { + resources.Limits = v1.ResourceList{} + } + resources.Limits[v1.ResourceMemory] = *resource.NewQuantity( + containerConfig.Linux.Resources.MemoryLimitInBytes, + resource.BinarySI, + ) + } + + pod.Spec.Containers[0].Resources = resources + } + + return pod +} +``` + +## CRI Interface Implementation + +### Runtime Service Methods + +#### Core Container Lifecycle + +```go +// Container creation with interLink integration +func (f *RemoteRuntime) CreateContainer( + ctx context.Context, + req *kubeapi.CreateContainerRequest +) (*kubeapi.CreateContainerResponse, error) { + // 1. Create in fake runtime for CRI compatibility + containerID, err := f.RuntimeService.CreateContainer( + ctx, req.PodSandboxId, req.Config, req.SandboxConfig + ) + + // 2. Convert to interLink format and track + if f.InterLinkHandler != nil { + pod := f.convertToInterLinkPod(ctx, req.SandboxConfig, req.Config) + podStatus := types.PodStatus{ + PodName: pod.Name, + PodNamespace: pod.Namespace, + PodUID: string(pod.UID), + Containers: []v1.ContainerStatus{}, + } + api.PodStatuses.Statuses[string(pod.UID)] = podStatus + } + + return &kubeapi.CreateContainerResponse{ContainerId: containerID}, nil +} +``` + +#### Container State Management + +```go +func (f *RemoteRuntime) StartContainer( + ctx context.Context, + req *kubeapi.StartContainerRequest +) (*kubeapi.StartContainerResponse, error) { + // Start in fake runtime + err := f.RuntimeService.StartContainer(ctx, req.ContainerId) + + // Update interLink tracking + if f.InterLinkHandler != nil { + f.RuntimeService.Lock() + container, exists := f.RuntimeService.Containers[req.ContainerId] + f.RuntimeService.Unlock() + + if exists { + containerStatus := v1.ContainerStatus{ + Name: container.Metadata.Name, + Image: container.Image.Image, + State: v1.ContainerState{ + Running: &v1.ContainerStateRunning{ + StartedAt: metav1.NewTime(time.Now()), + }, + }, + Ready: true, + } + updatePodStatusContainer(container.SandboxID, containerStatus) + } + } + + return &kubeapi.StartContainerResponse{}, nil +} +``` + +### Pod Sandbox Management + +```go +func (f *RemoteRuntime) RemovePodSandbox( + ctx context.Context, + req *kubeapi.RemovePodSandboxRequest +) (*kubeapi.RemovePodSandboxResponse, error) { + // Clean up interLink tracking + if f.InterLinkHandler != nil { + if podStatus, found := getPodStatus(req.PodSandboxId); found { + // Create deletion object for interLink + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podStatus.PodName, + Namespace: podStatus.PodNamespace, + UID: ktypes.UID(podStatus.PodUID), + }, + } + // Clean up tracking + removePodStatus(req.PodSandboxId) + } + } + + // Remove from fake runtime + return &kubeapi.RemovePodSandboxResponse{}, + f.RuntimeService.StopPodSandbox(ctx, req.PodSandboxId) +} +``` + +## Integration Patterns + +### 1. Dual-Layer Architecture + +The implementation uses a dual-layer approach: + +- **CRI Layer**: Maintains compatibility with kubelet expectations +- **InterLink Layer**: Provides actual container execution via plugins + +```go +// CRI compatibility +containerID, err := f.RuntimeService.CreateContainer(...) + +// InterLink integration +if f.InterLinkHandler != nil { + // Convert and track in interLink + pod := f.convertToInterLinkPod(...) + // Store in PodStatuses for lifecycle management +} +``` + +### 2. State Synchronization + +Container states are synchronized between CRI and interLink: + +```go +// Container state in CRI format +criState := &kubeapi.ContainerState{...} + +// Container state in Kubernetes format (for interLink) +k8sState := v1.ContainerState{ + Running: &v1.ContainerStateRunning{ + StartedAt: metav1.NewTime(time.Now()), + }, +} +``` + +### 3. Resource Mapping + +Resources are mapped from CRI limits to Kubernetes resource requirements: + +```go +// CRI resource limits +cpuQuota := containerConfig.Linux.Resources.CpuQuota +memoryLimit := containerConfig.Linux.Resources.MemoryLimitInBytes + +// Kubernetes resource requirements +resources := v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(cpuQuota/1000, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memoryLimit, resource.BinarySI), + }, +} +``` + +## Thread Safety + +### Mutex Usage + +The implementation handles concurrent access through multiple mutex layers: + +1. **FakeRuntimeService**: Internal mutex for CRI state +2. **PodStatuses**: Global mutex for interLink tracking (accessed via helpers) +3. **InterLinkHandler**: HTTP client safety + +### Safe Access Patterns + +```go +// Safe container access +f.RuntimeService.Lock() +container, exists := f.RuntimeService.Containers[containerID] +f.RuntimeService.Unlock() + +// Safe pod status access (via helpers) +podStatus, found := getPodStatus(sandboxID) +updatePodStatusContainer(sandboxID, containerStatus) +``` + +## Error Handling + +### Graceful Degradation + +The implementation gracefully handles missing components: + +```go +// InterLink integration is optional +if f.InterLinkHandler != nil { + // Perform interLink operations + // Errors here don't fail the CRI operation +} + +// CRI operation always completes +return &kubeapi.CreateContainerResponse{ContainerId: containerID}, nil +``` + +### Error Propagation + +```go +// CRI errors are propagated to kubelet +if err != nil { + return nil, fmt.Errorf("failed to create container: %w", err) +} + +// InterLink errors are logged but don't fail CRI operations +if interlinkErr != nil { + log.G(ctx).Warning("InterLink operation failed: ", interlinkErr) + // Continue with CRI response +} +``` + +## Performance Considerations + +### Memory Usage + +- **Container Tracking**: O(n) memory for n containers +- **Pod Status**: Centralized tracking with cleanup on removal +- **gRPC Buffers**: Standard gRPC memory patterns + +### Network Efficiency + +- **HTTP Keep-Alive**: Reused connections to interLink API +- **Batched Updates**: Status updates batched where possible +- **Local State**: CRI state maintained locally for fast responses + +### Scalability + +- **Concurrent Operations**: Thread-safe design supports concurrent CRI calls +- **Plugin Independence**: CRI operations don't block on plugin responses +- **Resource Limits**: Configurable timeouts and limits + +## Debugging and Observability + +### Logging Integration + +```go +import "github.com/containerd/containerd/log" + +// Structured logging with context +log.G(ctx).Info("Container created", + "containerID", containerID, + "podName", pod.Name, + "namespace", pod.Namespace) +``` + +### Tracing Support + +The implementation integrates with interLink's OpenTelemetry tracing: + +```go +tracer := otel.Tracer("interlink-cri") +_, span := tracer.Start(ctx, "CreateContainer") +defer span.End() + +span.SetAttributes( + attribute.String("container.name", containerConfig.Metadata.Name), + attribute.String("pod.uid", sandboxConfig.Metadata.Uid), +) +``` + +### Metrics + +Key metrics tracked: +- Container creation/deletion rates +- State transition timings +- Error rates by operation type +- Resource utilization + +## Extension Points + +### Custom Resource Mapping + +```go +// Extend convertToInterLinkPod for custom resources +if customResource := containerConfig.Annotations["custom-resource"]; customResource != "" { + // Handle custom resource types + pod.Spec.Containers[0].Resources.Limits["example.com/gpu"] = + resource.MustParse(customResource) +} +``` + +### Plugin-Specific Annotations + +```go +// Pass plugin-specific metadata through annotations +pod.Annotations = map[string]string{ + "interlink.io/plugin": "slurm", + "interlink.io/partition": "gpu", + "interlink.io/qos": "high", +} +``` + +### Custom State Handlers + +```go +// Extend status mapping for custom states +func mapCustomState(interlinkState types.ContainerState) *kubeapi.ContainerState { + // Custom state mapping logic + return criState +} +``` + +## Testing Considerations + +### Unit Testing + +```go +func TestCreateContainer(t *testing.T) { + // Mock InterLinkHandler + mockHandler := &api.InterLinkHandler{...} + + // Create runtime with mock + runtime := NewFakeRemoteRuntime(mockHandler) + + // Test container creation + req := &kubeapi.CreateContainerRequest{...} + resp, err := runtime.CreateContainer(context.Background(), req) + + // Verify CRI response + assert.NoError(t, err) + assert.NotEmpty(t, resp.ContainerId) + + // Verify interLink integration + podStatus, found := getPodStatus(req.PodSandboxId) + assert.True(t, found) + assert.Equal(t, expectedPodName, podStatus.PodName) +} +``` + +### Integration Testing + +```go +func TestEndToEndWorkflow(t *testing.T) { + // Start test interLink server + testServer := startTestInterLinkServer(t) + defer testServer.Close() + + // Create CRI runtime + runtime := NewFakeRemoteRuntime(createTestHandler(testServer.URL)) + + // Test full container lifecycle + // 1. Create sandbox + // 2. Create container + // 3. Start container + // 4. Check status + // 5. Stop container + // 6. Remove container + // 7. Remove sandbox +} +``` + +## Future Enhancements + +### Planned Features + +1. **Advanced State Mapping**: More sophisticated CRI ↔ interLink state mapping +2. **Resource Scheduling**: Integration with Kubernetes scheduler +3. **Volume Support**: Enhanced volume mounting capabilities +4. **Security Contexts**: Full security context support +5. **Network Policies**: Advanced networking integration + +### Extension Opportunities + +1. **Custom Plugins**: Easy plugin development framework +2. **Multi-Cluster**: Cross-cluster container execution +3. **Hybrid Runtimes**: Mixed local/remote execution +4. **AI/ML Workloads**: Specialized support for AI/ML containers + +This technical reference provides the foundation for understanding, extending, and troubleshooting the interLink CRI implementation. \ No newline at end of file diff --git a/docs/docs/other-integrations/cri/_category_.json b/docs/docs/other-integrations/cri/_category_.json new file mode 100644 index 00000000..ae352c2b --- /dev/null +++ b/docs/docs/other-integrations/cri/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Container Runtime Interface (CRI)", + "position": 1, + "link": { + "type": "generated-index", + "description": "Use interLink as a Container Runtime Interface (CRI) for direct kubelet integration." + } +} \ No newline at end of file diff --git a/docs/versions.json b/docs/versions.json index 18358f59..75062ec7 100644 --- a/docs/versions.json +++ b/docs/versions.json @@ -1,4 +1,4 @@ [ - "0.4.x", - "0.5.x" + "0.5.x", + "0.4.x" ] diff --git a/pkg/virtualkubelet/cert-retriever.go b/pkg/virtualkubelet/cert-retriever.go index 227d7e12..c8a2905e 100644 --- a/pkg/virtualkubelet/cert-retriever.go +++ b/pkg/virtualkubelet/cert-retriever.go @@ -143,3 +143,57 @@ func NewSelfSignedCertificateRetriever(nodeName string, nodeIP net.IP) Crtretrie return cert, nil } } + +// NewCSRCertificateRetriever creates a certificate retriever that uses Kubernetes CSR API +// for more native certificate management. This function creates CSRs, submits them to the +// Kubernetes API server, and automatically manages certificate lifecycle. +func NewCSRCertificateRetriever(kubeClient kubernetes.Interface, nodeName string, nodeIP net.IP) (Crtretriever, error) { + const ( + vkCertsPath = "/tmp/certs" + vkCertsPrefix = "virtual-kubelet-csr" + ) + + certificateStore, err := certificate.NewFileStore(vkCertsPrefix, vkCertsPath, vkCertsPath, "", "") + if err != nil { + return nil, fmt.Errorf("failed to initialize CSR certificate store: %w", err) + } + + getTemplate := func() *x509.CertificateRequest { + return &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organization: []string{"system:nodes"}, + }, + IPAddresses: []net.IP{nodeIP}, + DNSNames: []string{nodeName, fmt.Sprintf("%s.local", nodeName)}, + } + } + + // Use kubernetes.io/kubelet-serving signer for virtual kubelet server certificates + mgr, err := certificate.NewManager(&certificate.Config{ + ClientsetFn: func(_ *tls.Certificate) (kubernetes.Interface, error) { + return kubeClient, nil + }, + GetTemplate: getTemplate, + SignerName: "kubernetes.io/kubelet-serving", + Usages: []certificates.KeyUsage{ + certificates.UsageDigitalSignature, + certificates.UsageKeyEncipherment, + certificates.UsageServerAuth, + }, + CertificateStore: certificateStore, + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize CSR certificate manager: %w", err) + } + + mgr.Start() + + return func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert := mgr.Current() + if cert == nil { + return nil, fmt.Errorf("no serving certificate available from CSR") + } + return cert, nil + }, nil +} diff --git a/pkg/virtualkubelet/config.go b/pkg/virtualkubelet/config.go index f1b604cd..0374108c 100644 --- a/pkg/virtualkubelet/config.go +++ b/pkg/virtualkubelet/config.go @@ -23,6 +23,7 @@ type Config struct { NodeLabels []string `yaml:"NodeLabels"` NodeTaints []TaintSpec `yaml:"NodeTaints"` TLS TLSConfig `yaml:"TLS,omitempty"` + KubeletHTTPSCertMode string `yaml:"KubeletHTTPSCertMode,omitempty"` // "self-signed" or "csr" } // TLSConfig holds TLS/mTLS configuration for secure communication with interLink API