trust the in-cluster oauth-server's certificate when constructing an OpenShift client

Author: stlaz

providers/openshift/authentication.go

@@ -13,8 +13,6 @@ import (
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
 	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 type RequestHeaderAuthenticationOptions struct {
@@ -190,33 +188,8 @@ func deserializeStrings(in string) ([]string, error) {
 	return ret, nil
 }
 
-func (s *DelegatingAuthenticationOptions) getClientConfig() (*rest.Config, error) {
-	var clientConfig *rest.Config
-	var err error
-	if len(s.RemoteKubeConfigFile) > 0 {
-		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.RemoteKubeConfigFile}
-		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
-
-		clientConfig, err = loader.ClientConfig()
-
-	} else {
-		// without the remote kubeconfig file, try to use the in-cluster config.  Most addon API servers will
-		// use this path
-		clientConfig, err = rest.InClusterConfig()
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	// set high qps/burst limits since this will effectively limit API server responsiveness
-	clientConfig.QPS = 200
-	clientConfig.Burst = 400
-
-	return clientConfig, nil
-}
-
 func (s *DelegatingAuthenticationOptions) newTokenAccessReview() (authenticationclient.TokenReviewInterface, error) {
-	clientConfig, err := s.getClientConfig()
+	clientConfig, err := GetClientConfig(s.RemoteKubeConfigFile)
 	if err != nil {
 		return nil, err
 	}

providers/openshift/authorization.go

@@ -8,8 +8,6 @@ import (
 
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 // DelegatingAuthorizationOptions provides an easy way for composing API servers to delegate their authorization to
@@ -69,27 +67,11 @@ func (s *DelegatingAuthorizationOptions) ToAuthorizationConfig() (authorizerfact
 }
 
 func (s *DelegatingAuthorizationOptions) newSubjectAccessReview() (authorizationclient.SubjectAccessReviewInterface, error) {
-	var clientConfig *rest.Config
-	var err error
-	if len(s.RemoteKubeConfigFile) > 0 {
-		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.RemoteKubeConfigFile}
-		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
-
-		clientConfig, err = loader.ClientConfig()
-
-	} else {
-		// without the remote kubeconfig file, try to use the in-cluster config.  Most addon API servers will
-		// use this path
-		clientConfig, err = rest.InClusterConfig()
-	}
+	clientConfig, err := GetClientConfig(s.RemoteKubeConfigFile)
 	if err != nil {
 		return nil, err
 	}
 
-	// set high qps/burst limits since this will effectively limit API server responsiveness
-	clientConfig.QPS = 200
-	clientConfig.Burst = 400
-
 	client, err := authorizationclient.NewForConfig(clientConfig)
 	if err != nil {
 		return nil, err

providers/openshift/kubeclient.go

@@ -0,0 +1,39 @@
+package openshift
+
+import (
+	"flag"
+
+	"k8s.io/client-go/kubernetes"
+)
+
+type KubeClientOptions struct {
+	// RemoteKubeConfigFile is the file to use to connect to a "normal" kube API server
+	RemoteKubeConfigFile string
+}
+
+func NewKubeClientOptions() *KubeClientOptions {
+	return &KubeClientOptions{}
+}
+
+func (o *KubeClientOptions) Validate() []error {
+	return []error{}
+}
+
+func (o *KubeClientOptions) AddFlags(fs *flag.FlagSet) {
+	fs.StringVar(&o.RemoteKubeConfigFile, "kubeconfig", o.RemoteKubeConfigFile, ""+
+		"kubeconfig file pointing at the 'core' OpenShift server that has the oauth-server running on it")
+}
+
+func (o *KubeClientOptions) ToKubeClientConfig() (*kubernetes.Clientset, error) {
+	clientConfig, err := GetClientConfig(o.RemoteKubeConfigFile)
+	if err != nil {
+		return nil, err
+	}
+
+	kubeClient, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return kubeClient, nil
+}

providers/openshift/provider.go

@@ -28,9 +28,14 @@ import (
 	authenticationv1 "k8s.io/api/authentication/v1"
 	authorizationv1 "k8s.io/api/authorization/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/client-go/informers"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 func emptyURL(u *url.URL) bool {
@@ -45,13 +50,15 @@ type OpenShiftProvider struct {
 
 	AuthenticationOptions DelegatingAuthenticationOptions
 	AuthorizationOptions  DelegatingAuthorizationOptions
+	KubeClientOptions     KubeClientOptions
 
-	authenticator authenticator.Request
-	authorizer    authorizer.Authorizer
-	defaultRecord authorizer.AttributesRecord
-	reviews       []string
-	paths         recordsByPath
-	hostreviews   map[string][]string
+	authenticator   authenticator.Request
+	authorizer      authorizer.Authorizer
+	configMapLister corev1listers.ConfigMapLister
+	defaultRecord   authorizer.AttributesRecord
+	reviews         []string
+	paths           recordsByPath
+	hostreviews     map[string][]string
 
 	// httpClientCache stores httpClient objects so that new client does not have to
 	// be created on each request just to prevent CAs content changed
@@ -83,6 +90,7 @@ func (p *OpenShiftProvider) SetClientCAFile(file string) {
 }
 
 func (p *OpenShiftProvider) Bind(flags *flag.FlagSet) {
+	p.KubeClientOptions.AddFlags(flags)
 	p.AuthenticationOptions.AddFlags(flags)
 	p.AuthorizationOptions.AddFlags(flags)
 }
@@ -145,7 +153,17 @@ func (p *OpenShiftProvider) newOpenShiftClient() (*http.Client, error) {
 
 	// try to retrieve a cached client
 	metadataHash, err := util.GetFilesMetadataHash(capaths)
-	if httpClient, ok := p.httpClientCache.Load(metadataHash); ok {
+	if err != nil {
+		return nil, err
+	}
+
+	oauthServerCert, err := p.configMapLister.ConfigMaps("openshift-config-managed").Get("oauth-serving-cert")
+	if err != nil {
+		return nil, err
+	}
+
+	cachedKey := metadataHash + oauthServerCert.ResourceVersion
+	if httpClient, ok := p.httpClientCache.Load(cachedKey); ok {
 		return httpClient.(*http.Client), nil
 	}
 
@@ -155,6 +173,10 @@ func (p *OpenShiftProvider) newOpenShiftClient() (*http.Client, error) {
 		return nil, err
 	}
 
+	if ok := pool.AppendCertsFromPEM([]byte(oauthServerCert.Data["ca-bundle.crt"])); !ok {
+		log.Println("failed to add the oauth-server certificate to the OpenShift client CA bundle")
+	}
+
 	httpClient := &http.Client{
 		Jar: http.DefaultClient.Jar,
 		Transport: &http.Transport{
@@ -163,7 +185,7 @@ func (p *OpenShiftProvider) newOpenShiftClient() (*http.Client, error) {
 		},
 		Timeout: 1 * time.Minute,
 	}
-	p.httpClientCache.Store(metadataHash, httpClient)
+	p.httpClientCache.Store(cachedKey, httpClient)
 
 	return httpClient, nil
 }
@@ -310,6 +332,21 @@ func (p *OpenShiftProvider) Complete(data *providers.ProviderData, reviewURL *ur
 	p.ProviderData = data
 	p.ReviewURL = reviewURL
 
+	kubeClient, err := p.KubeClientOptions.ToKubeClientConfig()
+	if err != nil {
+		return err
+	}
+
+	kubeInformersMachineConfigNS := informers.NewSharedInformerFactoryWithOptions(
+		kubeClient,
+		10*time.Minute,
+		informers.WithNamespace("openshift-config-managed"),
+		informers.WithTweakListOptions(func(opts *metav1.ListOptions) {
+			opts.FieldSelector = fields.OneTermEqualSelector("metadata.name", "oauth-serving-cert").String()
+		}))
+	go kubeInformersMachineConfigNS.Core().V1().ConfigMaps().Informer().Run(context.TODO().Done())
+	p.configMapLister = kubeInformersMachineConfigNS.Core().V1().ConfigMaps().Lister()
+
 	if len(p.paths) > 0 {
 		log.Printf("Delegation of authentication and authorization to OpenShift is enabled for bearer tokens and client certificates.")
 
@@ -655,3 +692,28 @@ func getKubeAPIURLWithPath(path string) *url.URL {
 
 	return ret
 }
+
+func GetClientConfig(remoteKubeConfigFile string) (*rest.Config, error) {
+	var clientConfig *rest.Config
+	var err error
+	if len(remoteKubeConfigFile) > 0 {
+		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: remoteKubeConfigFile}
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+
+		clientConfig, err = loader.ClientConfig()
+
+	} else {
+		// without the remote kubeconfig file, try to use the in-cluster config.  Most addon API servers will
+		// use this path
+		clientConfig, err = rest.InClusterConfig()
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// set high qps/burst limits since this will effectively limit API server responsiveness
+	clientConfig.QPS = 200
+	clientConfig.Burst = 400
+
+	return clientConfig, nil
+}

providers/openshift/provider_test.go

@@ -8,9 +8,15 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
 )
 
 type mockAuthRequestHandler struct {
@@ -147,7 +153,19 @@ func TestNewOpenShiftClient(t *testing.T) {
 		t.Fatalf("failed to write CA cert to tmpfile: %v", err)
 	}
 
-	p := &OpenShiftProvider{}
+	indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{})
+	err = indexer.Add(&corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "oauth-serving-cert",
+			Namespace: "openshift-config-managed",
+		},
+	})
+	require.NoError(t, err)
+
+	p := &OpenShiftProvider{
+		configMapLister: corev1listers.NewConfigMapLister(indexer),
+	}
+
 	p.paths = recordsByPath{pathRecord{"/someurl", authorizer.AttributesRecord{}}}
 	p.authenticator = &mockAuthRequestHandler{}
 	p.authorizer = &mockAuthorizer{}