Skip to content

Commit 0637b9d

Browse files
OCPNODE-3201: Create MC to preserve 4.20 kubeletconfig post upgrade
1 parent 1f11aef commit 0637b9d

File tree

7 files changed

+185
-13
lines changed

7 files changed

+185
-13
lines changed

pkg/controller/bootstrap/bootstrap.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,13 @@ func (b *Bootstrap) Run(destDir string) error {
208208
}
209209
klog.Infof("Successfully generated MachineConfigs from feature gates.")
210210

211+
compressibleConfigs, err := kubeletconfig.RunCompressibleBootstrap(pools, cconfig, b.templatesDir, apiServer, fgHandler)
212+
if err != nil {
213+
return err
214+
}
215+
configs = append(configs, compressibleConfigs...)
216+
klog.Infof("Successfully generated MachineConfigs for compressible kubelet configs.")
217+
211218
if nodeConfig == nil {
212219
nodeConfig = &apicfgv1.Node{
213220
ObjectMeta: metav1.ObjectMeta{

pkg/controller/kubelet-config/kubelet_config_bootstrap.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func RunKubeletBootstrap(templateDir string, kubeletConfigs []*mcfgv1.KubeletCon
4040
}
4141
role := pool.Name
4242

43-
originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, fgHandler, apiServer)
43+
originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(controllerConfig, templateDir, role, fgHandler, apiServer)
4444
if err != nil {
4545
return nil, err
4646
}
Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
package kubeletconfig
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
"github.com/clarketm/json"
8+
configv1 "github.com/openshift/api/config/v1"
9+
mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
10+
ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
11+
"k8s.io/apimachinery/pkg/api/errors"
12+
"k8s.io/apimachinery/pkg/labels"
13+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
14+
"k8s.io/client-go/util/retry"
15+
"k8s.io/klog/v2"
16+
)
17+
18+
const (
19+
// CompressibleMachineConfigNamePrefix is the prefix for compressible machine configs
20+
CompressibleMachineConfigNamePrefix = "50-%s-compressible-kubelet-override"
21+
22+
// KubeletConfPath is the path to the kubelet config file
23+
KubeletConfPath = "/etc/kubernetes/kubelet.conf"
24+
)
25+
26+
// ensureCompressibleMachineConfigs ensures compressible machine configs exist for all pools
27+
// This is called at controller startup to create compressible MCs for all pools
28+
func (ctrl *Controller) ensureCompressibleMachineConfigs() error {
29+
// Get all pools
30+
pools, err := ctrl.mcpLister.List(labels.Everything())
31+
if err != nil {
32+
return fmt.Errorf("could not list machine config pools: %w", err)
33+
}
34+
35+
// Get ControllerConfig
36+
cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName)
37+
if err != nil {
38+
return fmt.Errorf("could not get ControllerConfig: %w", err)
39+
}
40+
41+
// Get APIServer
42+
apiServer, err := ctrl.apiserverLister.Get(ctrlcommon.APIServerInstanceName)
43+
if err != nil && !errors.IsNotFound(err) {
44+
return fmt.Errorf("could not get APIServer: %w", err)
45+
}
46+
47+
// Create compressible MC for each pool
48+
for _, pool := range pools {
49+
// Generate the original kubelet config for this pool
50+
_, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, pool.Name, ctrl.fgHandler, apiServer)
51+
if err != nil {
52+
klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err)
53+
continue
54+
}
55+
56+
// Create compressible MC
57+
if err := ctrl.createCompressibleMachineConfigIfNeeded(pool.Name, kubeletContents); err != nil {
58+
klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err)
59+
// Don't fail startup if compressible MC creation fails for a pool
60+
continue
61+
}
62+
}
63+
64+
return nil
65+
}
66+
67+
// createCompressibleMachineConfigIfNeeded creates a compressible machine config if it doesn't exist
68+
// This function is called from the controller after kubelet config is successfully generated
69+
func (ctrl *Controller) createCompressibleMachineConfigIfNeeded(poolName string, kubeletContents []byte) error {
70+
compressibleKey := fmt.Sprintf(CompressibleMachineConfigNamePrefix, poolName)
71+
_, err := ctrl.client.MachineconfigurationV1().MachineConfigs().Get(context.TODO(), compressibleKey, metav1.GetOptions{})
72+
compressibleIsNotFound := errors.IsNotFound(err)
73+
if err != nil && !compressibleIsNotFound {
74+
return err
75+
}
76+
77+
if compressibleIsNotFound {
78+
compressibleMC, err := newCompressibleMachineConfig(poolName, kubeletContents)
79+
if err != nil {
80+
return fmt.Errorf("could not create compressible machine config: %w", err)
81+
}
82+
83+
if err := retry.RetryOnConflict(updateBackoff, func() error {
84+
_, err := ctrl.client.MachineconfigurationV1().MachineConfigs().Create(context.TODO(), compressibleMC, metav1.CreateOptions{})
85+
return err
86+
}); err != nil {
87+
return fmt.Errorf("could not create compressible MachineConfig: %w", err)
88+
}
89+
klog.Infof("Created compressible kubelet configuration %v for pool %v", compressibleKey, poolName)
90+
} else {
91+
klog.V(4).Infof("Compressible kubelet MachineConfig %v already exists for pool %v, skipping creation", compressibleKey, poolName)
92+
}
93+
94+
return nil
95+
}
96+
97+
// RunCompressibleBootstrap generates compressible machine configs for all pools during bootstrap
98+
func RunCompressibleBootstrap(pools []*mcfgv1.MachineConfigPool, cconfig *mcfgv1.ControllerConfig, templatesDir string, apiServer *configv1.APIServer, fgHandler ctrlcommon.FeatureGatesHandler) ([]*mcfgv1.MachineConfig, error) {
99+
configs := []*mcfgv1.MachineConfig{}
100+
101+
for _, pool := range pools {
102+
// Generate the original kubelet config for this pool
103+
_, kubeletContents, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templatesDir, pool.Name, fgHandler, apiServer)
104+
if err != nil {
105+
klog.Warningf("Failed to generate kubelet config for pool %v: %v", pool.Name, err)
106+
continue
107+
}
108+
109+
// Create compressible MC
110+
compressibleMC, err := newCompressibleMachineConfig(pool.Name, kubeletContents)
111+
if err != nil {
112+
klog.Warningf("Failed to create compressible machine config for pool %v: %v", pool.Name, err)
113+
continue
114+
}
115+
116+
configs = append(configs, compressibleMC)
117+
}
118+
119+
return configs, nil
120+
}
121+
122+
// newCompressibleMachineConfig creates a new machine config for compressible kubelet override
123+
// from the provided kubelet config contents
124+
func newCompressibleMachineConfig(poolName string, kubeletContents []byte) (*mcfgv1.MachineConfig, error) {
125+
compressibleMCName := fmt.Sprintf(CompressibleMachineConfigNamePrefix, poolName)
126+
ignConfig := ctrlcommon.NewIgnConfig()
127+
compressibleMC, err := ctrlcommon.MachineConfigFromIgnConfig(poolName, compressibleMCName, ignConfig)
128+
if err != nil {
129+
return nil, fmt.Errorf("could not create machine config from ignition config: %w", err)
130+
}
131+
132+
rawCompressibleIgn, err := createCompressibleKubeletIgnConfig(kubeletContents)
133+
if err != nil {
134+
return nil, fmt.Errorf("could not create compressible kubelet ignition config: %w", err)
135+
}
136+
137+
compressibleMC.Spec.Config.Raw = rawCompressibleIgn
138+
compressibleMC.ObjectMeta.Annotations = map[string]string{
139+
"openshift-patch-reference": "machineConfig-to-override-kubelet-conf-for-compressible-resources",
140+
}
141+
142+
return compressibleMC, nil
143+
}
144+
145+
// createCompressibleKubeletIgnConfig creates an Ignition config that overrides /etc/kubernetes/kubelet.conf
146+
// from the provided kubelet config contents
147+
func createCompressibleKubeletIgnConfig(kubeletContents []byte) ([]byte, error) {
148+
// Create an Ignition file that overrides /etc/kubernetes/kubelet.conf
149+
compressibleFile := ctrlcommon.NewIgnFileBytesOverwriting(KubeletConfPath, kubeletContents)
150+
compressibleIgnConfig := ctrlcommon.NewIgnConfig()
151+
compressibleIgnConfig.Storage.Files = append(compressibleIgnConfig.Storage.Files, compressibleFile)
152+
153+
rawCompressibleIgn, err := json.Marshal(compressibleIgnConfig)
154+
if err != nil {
155+
return nil, fmt.Errorf("could not marshal ignition config: %w", err)
156+
}
157+
158+
return rawCompressibleIgn, nil
159+
}

pkg/controller/kubelet-config/kubelet_config_controller.go

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,11 @@ func (ctrl *Controller) Run(workers int, stopCh <-chan struct{}) {
200200
klog.Info("Starting MachineConfigController-KubeletConfigController")
201201
defer klog.Info("Shutting down MachineConfigController-KubeletConfigController")
202202

203+
// Ensure compressible machine configs are created for all pools at startup
204+
if err := ctrl.ensureCompressibleMachineConfigs(); err != nil {
205+
klog.Warningf("Error ensuring compressible MachineConfigs: %v", err)
206+
}
207+
203208
for i := 0; i < workers; i++ {
204209
go wait.Until(ctrl.worker, time.Second, stopCh)
205210
}
@@ -419,21 +424,22 @@ func (ctrl *Controller) handleFeatureErr(err error, key string) {
419424

420425
// generateOriginalKubeletConfigWithFeatureGates generates a KubeletConfig and ensure the correct feature gates are set
421426
// based on the given FeatureGate.
422-
func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer) (*kubeletconfigv1beta1.KubeletConfiguration, error) {
427+
// It also returns the decoded kubelet config contents for use in creating compressible machine configs.
428+
func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, apiServer *configv1.APIServer) (*kubeletconfigv1beta1.KubeletConfiguration, []byte, error) {
423429
originalKubeletIgn, err := generateOriginalKubeletConfigIgn(cc, templatesDir, role, apiServer)
424430
if err != nil {
425-
return nil, fmt.Errorf("could not generate the original Kubelet config ignition: %w", err)
431+
return nil, nil, fmt.Errorf("could not generate the original Kubelet config ignition: %w", err)
426432
}
427433
if originalKubeletIgn.Contents.Source == nil {
428-
return nil, fmt.Errorf("the original Kubelet source string is empty: %w", err)
434+
return nil, nil, fmt.Errorf("the original Kubelet source string is empty: %w", err)
429435
}
430436
contents, err := ctrlcommon.DecodeIgnitionFileContents(originalKubeletIgn.Contents.Source, originalKubeletIgn.Contents.Compression)
431437
if err != nil {
432-
return nil, fmt.Errorf("could not decode the original Kubelet source string: %w", err)
438+
return nil, nil, fmt.Errorf("could not decode the original Kubelet source string: %w", err)
433439
}
434440
originalKubeConfig, err := DecodeKubeletConfig(contents)
435441
if err != nil {
436-
return nil, fmt.Errorf("could not deserialize the Kubelet source: %w", err)
442+
return nil, nil, fmt.Errorf("could not deserialize the Kubelet source: %w", err)
437443
}
438444

439445
// todo map pointer
@@ -442,10 +448,10 @@ func generateOriginalKubeletConfigWithFeatureGates(cc *mcfgv1.ControllerConfig,
442448
// Merge in Feature Gates.
443449
// If they are the same, this will be a no-op
444450
if err := mergo.Merge(&originalKubeConfig.FeatureGates, featureGates, mergo.WithOverride); err != nil {
445-
return nil, fmt.Errorf("could not merge feature gates: %w", err)
451+
return nil, nil, fmt.Errorf("could not merge feature gates: %w", err)
446452
}
447453

448-
return originalKubeConfig, nil
454+
return originalKubeConfig, contents, nil
449455
}
450456

451457
func generateOriginalKubeletConfigIgn(cc *mcfgv1.ControllerConfig, templatesDir, role string, apiServer *osev1.APIServer) (*ign3types.File, error) {
@@ -616,7 +622,7 @@ func (ctrl *Controller) syncKubeletConfig(key string) error {
616622
return fmt.Errorf("could not get ControllerConfig %w", err)
617623
}
618624

619-
originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer)
625+
originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer)
620626
if err != nil {
621627
return ctrl.syncStatusOnly(cfg, err, "could not get original kubelet config: %v", err)
622628
}

pkg/controller/kubelet-config/kubelet_config_features.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ func generateFeatureMap(fgHandler ctrlcommon.FeatureGatesHandler, exclusions ...
191191
}
192192

193193
func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir, role string, fgHandler ctrlcommon.FeatureGatesHandler, nodeConfig *osev1.Node, apiServer *osev1.APIServer) ([]byte, error) {
194-
originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer)
194+
originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, templatesDir, role, fgHandler, apiServer)
195195
if err != nil {
196196
return nil, err
197197
}

pkg/controller/kubelet-config/kubelet_config_features_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ func TestFeatureGateDrift(t *testing.T) {
4343
ctrl := f.newController(fgHandler)
4444

4545
// Generate kubelet config with feature gates applied
46-
kubeletConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil)
46+
kubeletConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, "master", fgHandler, nil)
4747
require.NoError(t, err)
4848

4949
t.Logf("Generated Kubelet Config Feature Gates: %v", kubeletConfig.FeatureGates)

pkg/controller/kubelet-config/kubelet_config_nodes.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ func (ctrl *Controller) syncNodeConfigHandler(key string) error {
112112
return err
113113
}
114114
}
115-
originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer)
115+
originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cc, ctrl.templatesDir, role, ctrl.fgHandler, apiServer)
116116
if err != nil {
117117
return err
118118
}
@@ -290,7 +290,7 @@ func RunNodeConfigBootstrap(templateDir string, fgHandler ctrlcommon.FeatureGate
290290
if err != nil {
291291
return nil, err
292292
}
293-
originalKubeConfig, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templateDir, role, fgHandler, apiServer)
293+
originalKubeConfig, _, err := generateOriginalKubeletConfigWithFeatureGates(cconfig, templateDir, role, fgHandler, apiServer)
294294
if err != nil {
295295
return nil, err
296296
}

0 commit comments

Comments
 (0)