Skip to content

Commit a4b83e7

Browse files
stlazstlaz
committed
trustbundles: add a new kube-apiserver-serving signer
1 parent 09e5e62 commit a4b83e7

File tree

10 files changed

+1179
-54
lines changed

10 files changed

+1179
-54
lines changed

‎cmd/kube-controller-manager/app/certificates.go‎

Lines changed: 84 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,21 @@ import (
2323
"context"
2424
"fmt"
2525

26+
certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
27+
"k8s.io/apiserver/pkg/server/dynamiccertificates"
28+
utilfeature "k8s.io/apiserver/pkg/util/feature"
29+
"k8s.io/client-go/kubernetes"
30+
"k8s.io/component-base/featuregate"
2631
"k8s.io/controller-manager/controller"
2732
"k8s.io/klog/v2"
2833
"k8s.io/kubernetes/cmd/kube-controller-manager/names"
2934
"k8s.io/kubernetes/pkg/controller/certificates/approver"
3035
"k8s.io/kubernetes/pkg/controller/certificates/cleaner"
36+
ctbpublisher "k8s.io/kubernetes/pkg/controller/certificates/clustertrustbundlepublisher"
3137
"k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher"
3238
"k8s.io/kubernetes/pkg/controller/certificates/signer"
3339
csrsigningconfig "k8s.io/kubernetes/pkg/controller/certificates/signer/config"
40+
"k8s.io/kubernetes/pkg/features"
3441
)
3542

3643
func newCertificateSigningRequestSigningControllerDescriptor() *ControllerDescriptor {
@@ -200,16 +207,9 @@ func newRootCACertificatePublisherControllerDescriptor() *ControllerDescriptor {
200207
}
201208

202209
func startRootCACertificatePublisherController(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
203-
var (
204-
rootCA []byte
205-
err error
206-
)
207-
if controllerContext.ComponentConfig.SAController.RootCAFile != "" {
208-
if rootCA, err = readCA(controllerContext.ComponentConfig.SAController.RootCAFile); err != nil {
209-
return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", controllerContext.ComponentConfig.SAController.RootCAFile, err)
210-
}
211-
} else {
212-
rootCA = controllerContext.ClientBuilder.ConfigOrDie("root-ca-cert-publisher").CAData
210+
rootCA, err := getKubeAPIServerCAFileContents(controllerContext)
211+
if err != nil {
212+
return nil, true, err
213213
}
214214

215215
sac, err := rootcacertpublisher.NewPublisher(
@@ -224,3 +224,77 @@ func startRootCACertificatePublisherController(ctx context.Context, controllerCo
224224
go sac.Run(ctx, 1)
225225
return nil, true, nil
226226
}
227+
228+
func newKubeAPIServerSignerClusterTrustBundledPublisherDescriptor() *ControllerDescriptor {
229+
return &ControllerDescriptor{
230+
name: names.KubeAPIServerClusterTrustBundlePublisherController,
231+
initFunc: newKubeAPIServerSignerClusterTrustBundledPublisherController,
232+
requiredFeatureGates: []featuregate.Feature{features.ClusterTrustBundle},
233+
}
234+
}
235+
236+
func newKubeAPIServerSignerClusterTrustBundledPublisherController(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
237+
rootCA, err := getKubeAPIServerCAFileContents(controllerContext)
238+
if err != nil {
239+
return nil, false, err
240+
}
241+
242+
if len(rootCA) == 0 || !utilfeature.DefaultFeatureGate.Enabled(features.ClusterTrustBundle) {
243+
return nil, false, nil
244+
}
245+
246+
apiserverSignerClient := controllerContext.ClientBuilder.ClientOrDie("kube-apiserver-serving-clustertrustbundle-publisher")
247+
ctbAvailable, err := clusterTrustBundlesAvailable(apiserverSignerClient)
248+
if err != nil {
249+
return nil, false, fmt.Errorf("discovery failed for ClusterTrustBundle: %w", err)
250+
}
251+
252+
if !ctbAvailable {
253+
return nil, false, nil
254+
}
255+
256+
servingSigners, err := dynamiccertificates.NewStaticCAContent("kube-apiserver-serving", rootCA)
257+
if err != nil {
258+
return nil, false, fmt.Errorf("failed to create a static CA content provider for the kube-apiserver-serving signer: %w", err)
259+
}
260+
261+
ctbPublisher, err := ctbpublisher.NewClusterTrustBundlePublisher(
262+
"kubernetes.io/kube-apiserver-serving",
263+
servingSigners,
264+
apiserverSignerClient,
265+
)
266+
if err != nil {
267+
return nil, false, fmt.Errorf("error creating kube-apiserver-serving signer certificates publisher: %w", err)
268+
}
269+
270+
go ctbPublisher.Run(ctx)
271+
return nil, true, nil
272+
}
273+
274+
func clusterTrustBundlesAvailable(client kubernetes.Interface) (bool, error) {
275+
resList, err := client.Discovery().ServerResourcesForGroupVersion(certificatesv1alpha1.SchemeGroupVersion.String())
276+
277+
if resList != nil {
278+
// even in case of an error above there might be a partial list for APIs that
279+
// were already successfully discovered
280+
for _, r := range resList.APIResources {
281+
if r.Name == "clustertrustbundles" {
282+
return true, nil
283+
}
284+
}
285+
}
286+
return false, err
287+
}
288+
289+
func getKubeAPIServerCAFileContents(controllerContext ControllerContext) ([]byte, error) {
290+
if controllerContext.ComponentConfig.SAController.RootCAFile == "" {
291+
return controllerContext.ClientBuilder.ConfigOrDie("root-ca-cert-publisher").CAData, nil
292+
}
293+
294+
rootCA, err := readCA(controllerContext.ComponentConfig.SAController.RootCAFile)
295+
if err != nil {
296+
return nil, fmt.Errorf("error parsing root-ca-file at %s: %w", controllerContext.ComponentConfig.SAController.RootCAFile, err)
297+
}
298+
return rootCA, nil
299+
300+
}

‎cmd/kube-controller-manager/app/controllermanager.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -570,6 +570,7 @@ func NewControllerDescriptors() map[string]*ControllerDescriptor {
570570
register(newVolumeAttributesClassProtectionControllerDescriptor())
571571
register(newTTLAfterFinishedControllerDescriptor())
572572
register(newRootCACertificatePublisherControllerDescriptor())
573+
register(newKubeAPIServerSignerClusterTrustBundledPublisherDescriptor())
573574
register(newEphemeralVolumeControllerDescriptor())
574575

575576
// feature gated

‎cmd/kube-controller-manager/app/controllermanager_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ func TestControllerNamesDeclaration(t *testing.T) {
8989
names.VolumeAttributesClassProtectionController,
9090
names.TTLAfterFinishedController,
9191
names.RootCACertificatePublisherController,
92+
names.KubeAPIServerClusterTrustBundlePublisherController,
9293
names.EphemeralVolumeController,
9394
names.StorageVersionGarbageCollectorController,
9495
names.ResourceClaimController,

‎cmd/kube-controller-manager/names/controller_names.go‎

Lines changed: 45 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -42,48 +42,49 @@ package names
4242
// 3.2. when defined flag's help mentions a controller name
4343
// 4. defining a new service account for a new controller (old controllers may have inconsistent service accounts to stay backwards compatible)
4444
const (
45-
ServiceAccountTokenController = "serviceaccount-token-controller"
46-
EndpointsController = "endpoints-controller"
47-
EndpointSliceController = "endpointslice-controller"
48-
EndpointSliceMirroringController = "endpointslice-mirroring-controller"
49-
ReplicationControllerController = "replicationcontroller-controller"
50-
PodGarbageCollectorController = "pod-garbage-collector-controller"
51-
ResourceQuotaController = "resourcequota-controller"
52-
NamespaceController = "namespace-controller"
53-
ServiceAccountController = "serviceaccount-controller"
54-
GarbageCollectorController = "garbage-collector-controller"
55-
DaemonSetController = "daemonset-controller"
56-
JobController = "job-controller"
57-
DeploymentController = "deployment-controller"
58-
ReplicaSetController = "replicaset-controller"
59-
HorizontalPodAutoscalerController = "horizontal-pod-autoscaler-controller"
60-
DisruptionController = "disruption-controller"
61-
StatefulSetController = "statefulset-controller"
62-
CronJobController = "cronjob-controller"
63-
CertificateSigningRequestSigningController = "certificatesigningrequest-signing-controller"
64-
CertificateSigningRequestApprovingController = "certificatesigningrequest-approving-controller"
65-
CertificateSigningRequestCleanerController = "certificatesigningrequest-cleaner-controller"
66-
TTLController = "ttl-controller"
67-
BootstrapSignerController = "bootstrap-signer-controller"
68-
TokenCleanerController = "token-cleaner-controller"
69-
NodeIpamController = "node-ipam-controller"
70-
NodeLifecycleController = "node-lifecycle-controller"
71-
TaintEvictionController = "taint-eviction-controller"
72-
PersistentVolumeBinderController = "persistentvolume-binder-controller"
73-
PersistentVolumeAttachDetachController = "persistentvolume-attach-detach-controller"
74-
PersistentVolumeExpanderController = "persistentvolume-expander-controller"
75-
ClusterRoleAggregationController = "clusterrole-aggregation-controller"
76-
PersistentVolumeClaimProtectionController = "persistentvolumeclaim-protection-controller"
77-
PersistentVolumeProtectionController = "persistentvolume-protection-controller"
78-
TTLAfterFinishedController = "ttl-after-finished-controller"
79-
RootCACertificatePublisherController = "root-ca-certificate-publisher-controller"
80-
EphemeralVolumeController = "ephemeral-volume-controller"
81-
StorageVersionGarbageCollectorController = "storageversion-garbage-collector-controller"
82-
ResourceClaimController = "resourceclaim-controller"
83-
LegacyServiceAccountTokenCleanerController = "legacy-serviceaccount-token-cleaner-controller"
84-
ValidatingAdmissionPolicyStatusController = "validatingadmissionpolicy-status-controller"
85-
VolumeAttributesClassProtectionController = "volumeattributesclass-protection-controller"
86-
ServiceCIDRController = "service-cidr-controller"
87-
StorageVersionMigratorController = "storage-version-migrator-controller"
88-
SELinuxWarningController = "selinux-warning-controller"
45+
ServiceAccountTokenController = "serviceaccount-token-controller"
46+
EndpointsController = "endpoints-controller"
47+
EndpointSliceController = "endpointslice-controller"
48+
EndpointSliceMirroringController = "endpointslice-mirroring-controller"
49+
ReplicationControllerController = "replicationcontroller-controller"
50+
PodGarbageCollectorController = "pod-garbage-collector-controller"
51+
ResourceQuotaController = "resourcequota-controller"
52+
NamespaceController = "namespace-controller"
53+
ServiceAccountController = "serviceaccount-controller"
54+
GarbageCollectorController = "garbage-collector-controller"
55+
DaemonSetController = "daemonset-controller"
56+
JobController = "job-controller"
57+
DeploymentController = "deployment-controller"
58+
ReplicaSetController = "replicaset-controller"
59+
HorizontalPodAutoscalerController = "horizontal-pod-autoscaler-controller"
60+
DisruptionController = "disruption-controller"
61+
StatefulSetController = "statefulset-controller"
62+
CronJobController = "cronjob-controller"
63+
CertificateSigningRequestSigningController = "certificatesigningrequest-signing-controller"
64+
CertificateSigningRequestApprovingController = "certificatesigningrequest-approving-controller"
65+
CertificateSigningRequestCleanerController = "certificatesigningrequest-cleaner-controller"
66+
TTLController = "ttl-controller"
67+
BootstrapSignerController = "bootstrap-signer-controller"
68+
TokenCleanerController = "token-cleaner-controller"
69+
NodeIpamController = "node-ipam-controller"
70+
NodeLifecycleController = "node-lifecycle-controller"
71+
TaintEvictionController = "taint-eviction-controller"
72+
PersistentVolumeBinderController = "persistentvolume-binder-controller"
73+
PersistentVolumeAttachDetachController = "persistentvolume-attach-detach-controller"
74+
PersistentVolumeExpanderController = "persistentvolume-expander-controller"
75+
ClusterRoleAggregationController = "clusterrole-aggregation-controller"
76+
PersistentVolumeClaimProtectionController = "persistentvolumeclaim-protection-controller"
77+
PersistentVolumeProtectionController = "persistentvolume-protection-controller"
78+
TTLAfterFinishedController = "ttl-after-finished-controller"
79+
RootCACertificatePublisherController = "root-ca-certificate-publisher-controller"
80+
KubeAPIServerClusterTrustBundlePublisherController = "kube-apiserver-serving-clustertrustbundle-publisher-controller"
81+
EphemeralVolumeController = "ephemeral-volume-controller"
82+
StorageVersionGarbageCollectorController = "storageversion-garbage-collector-controller"
83+
ResourceClaimController = "resourceclaim-controller"
84+
LegacyServiceAccountTokenCleanerController = "legacy-serviceaccount-token-cleaner-controller"
85+
ValidatingAdmissionPolicyStatusController = "validatingadmissionpolicy-status-controller"
86+
VolumeAttributesClassProtectionController = "volumeattributesclass-protection-controller"
87+
ServiceCIDRController = "service-cidr-controller"
88+
StorageVersionMigratorController = "storage-version-migrator-controller"
89+
SELinuxWarningController = "selinux-warning-controller"
8990
)

‎pkg/controller/certificates/clustertrustbundlepublisher/metrics.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package clustertrustbundlepublisher
18+
19+
import (
20+
"errors"
21+
"strconv"
22+
"sync"
23+
"time"
24+
25+
apierrors "k8s.io/apimachinery/pkg/api/errors"
26+
"k8s.io/component-base/metrics"
27+
"k8s.io/component-base/metrics/legacyregistry"
28+
)
29+
30+
// clustertrustbundlePublisher - subsystem name used by clustertrustbundle_publisher
31+
const (
32+
clustertrustbundlePublisher = "clustertrustbundle_publisher"
33+
)
34+
35+
var (
36+
syncCounter = metrics.NewCounterVec(
37+
&metrics.CounterOpts{
38+
Subsystem: clustertrustbundlePublisher,
39+
Name: "sync_total",
40+
Help: "Number of syncs that occurred in cluster trust bundle publisher.",
41+
StabilityLevel: metrics.ALPHA,
42+
},
43+
[]string{"code"},
44+
)
45+
syncLatency = metrics.NewHistogramVec(
46+
&metrics.HistogramOpts{
47+
Subsystem: clustertrustbundlePublisher,
48+
Name: "sync_duration_seconds",
49+
Help: "The time it took to sync a cluster trust bundle.",
50+
Buckets: metrics.ExponentialBuckets(0.001, 2, 15),
51+
StabilityLevel: metrics.ALPHA,
52+
},
53+
[]string{"code"},
54+
)
55+
)
56+
57+
func recordMetrics(start time.Time, err error) {
58+
code := "500"
59+
if err == nil {
60+
code = "200"
61+
} else {
62+
var statusErr apierrors.APIStatus
63+
if errors.As(err, &statusErr) && statusErr.Status().Code != 0 {
64+
code = strconv.Itoa(int(statusErr.Status().Code))
65+
}
66+
}
67+
syncLatency.WithLabelValues(code).Observe(time.Since(start).Seconds())
68+
syncCounter.WithLabelValues(code).Inc()
69+
}
70+
71+
var once sync.Once
72+
73+
func registerMetrics() {
74+
once.Do(func() {
75+
legacyregistry.MustRegister(syncCounter)
76+
legacyregistry.MustRegister(syncLatency)
77+
})
78+
}

0 commit comments

Comments
 (0)