Skip to content

Commit ddc5024

Browse files
zhujian7zhujian7
authored
✨ Allow Custom Signer signingCA to Specify Namespace in AddOnTemplate (open-cluster-management-io#747)
* Allow Custom Signer signingCA to Specify Namespace in AddOnTemplate Signed-off-by: zhujian <[email protected]> * Add e2e test for signer namespace Signed-off-by: zhujian <[email protected]> --------- Signed-off-by: zhujian <[email protected]>
1 parent 1ef5904 commit ddc5024

13 files changed

+140
-39
lines changed

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ require (
3232
k8s.io/kube-aggregator v0.31.3
3333
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6
3434
open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50
35-
open-cluster-management.io/api v0.15.1-0.20241126073717-05ff7c1affe8
35+
open-cluster-management.io/api v0.15.1-0.20241209025232-b62746ae96d4
3636
open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f
3737
sigs.k8s.io/cluster-inventory-api v0.0.0-20240730014211-ef0154379848
3838
sigs.k8s.io/controller-runtime v0.19.3

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -453,8 +453,8 @@ k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY
453453
k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
454454
open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50 h1:TXRd6OdGjArh6cwlCYOqlIcyx21k81oUIYj4rmHlYx0=
455455
open-cluster-management.io/addon-framework v0.11.1-0.20241129080247-57b1d2859f50/go.mod h1:tsBSNs9mGfVQQjXBnjgpiX6r0UM+G3iNfmzQgKhEfw4=
456-
open-cluster-management.io/api v0.15.1-0.20241126073717-05ff7c1affe8 h1:yKI2N8VN3zij+2O8kEOGfXBtZDs3pMey0BFfikgBpJM=
457-
open-cluster-management.io/api v0.15.1-0.20241126073717-05ff7c1affe8/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
456+
open-cluster-management.io/api v0.15.1-0.20241209025232-b62746ae96d4 h1:f6KU3t9s0PA6vXmAjB6A9sd52OqBqOFK2uAhk3UUBKs=
457+
open-cluster-management.io/api v0.15.1-0.20241209025232-b62746ae96d4/go.mod h1:9erZEWEn4bEqh0nIX2wA7f/s3KCuFycQdBrPrRzi0QM=
458458
open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f h1:zeC7QrFNarfK2zY6jGtd+mX+yDrQQmnH/J8A7n5Nh38=
459459
open-cluster-management.io/sdk-go v0.15.1-0.20241125015855-1536c3970f8f/go.mod h1:fi5WBsbC5K3txKb8eRLuP0Sim/Oqz/PHX18skAEyjiA=
460460
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 h1:2770sDpzrjjsAtVhSeUFseziht227YAWYHLGNM8QPwY=

‎manifests/cluster-manager/hub/0000_03_addon.open-cluster-management.io_addontemplates.crd.yaml

+8-4
Original file line numberDiff line numberDiff line change
@@ -372,15 +372,19 @@ spec:
372372
signingCA:
373373
description: |-
374374
SigningCA represents the reference of the secret on the hub cluster to sign the CSR
375-
the secret must be in the namespace where the addon-manager is located, and the secret
376-
type must be "kubernetes.io/tls"
375+
the secret type must be "kubernetes.io/tls"
377376
Note: The addon manager will not have permission to access the secret by default, so
378-
the user must grant the permission to the addon manager(by creating rolebinding for
379-
the addon-manager serviceaccount "addon-manager-controller-sa").
377+
the user must grant the permission to the addon manager(by creating rolebinding/clusterrolebinding
378+
for the addon-manager serviceaccount "addon-manager-controller-sa").
380379
properties:
381380
name:
382381
description: Name of the signing CA secret
383382
type: string
383+
namespace:
384+
description: Namespace of the signing CA secret, the
385+
namespace of the addon-manager will be used if it
386+
is not set.
387+
type: string
384388
required:
385389
- name
386390
type: object

‎pkg/addon/templateagent/registration.go

+8-3
Original file line numberDiff line numberDiff line change
@@ -285,18 +285,23 @@ func CustomSignerWithExpiry(
285285
if csr.Spec.SignerName != customSignerConfig.SignerName {
286286
return nil
287287
}
288-
caSecret, err := kubeclient.CoreV1().Secrets(AddonManagerNamespace()).Get(
288+
289+
secretNamespace := AddonManagerNamespace()
290+
if len(customSignerConfig.SigningCA.Namespace) != 0 {
291+
secretNamespace = customSignerConfig.SigningCA.Namespace
292+
}
293+
caSecret, err := kubeclient.CoreV1().Secrets(secretNamespace).Get(
289294
context.TODO(), customSignerConfig.SigningCA.Name, metav1.GetOptions{})
290295
if err != nil {
291296
utilruntime.HandleError(fmt.Errorf("get custome signer ca %s/%s failed, %v",
292-
AddonManagerNamespace(), customSignerConfig.SigningCA.Name, err))
297+
secretNamespace, customSignerConfig.SigningCA.Name, err))
293298
return nil
294299
}
295300

296301
caData, caKey, err := extractCAdata(caSecret.Data[corev1.TLSCertKey], caSecret.Data[corev1.TLSPrivateKeyKey])
297302
if err != nil {
298303
utilruntime.HandleError(fmt.Errorf("get ca %s/%s data failed, %v",
299-
AddonManagerNamespace(), customSignerConfig.SigningCA.Name, err))
304+
secretNamespace, customSignerConfig.SigningCA.Name, err))
300305
return nil
301306
}
302307
return utils.DefaultSignerWithExpiry(caKey, caData, duration)(csr)

‎pkg/addon/templateagent/registration_test.go

+62
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"context"
66
"fmt"
7+
"net"
78
"os"
89
"strings"
910
"testing"
@@ -13,12 +14,14 @@ import (
1314
"github.com/stretchr/testify/assert"
1415
certificatesv1 "k8s.io/api/certificates/v1"
1516
certificates "k8s.io/api/certificates/v1beta1"
17+
corev1 "k8s.io/api/core/v1"
1618
rbacv1 "k8s.io/api/rbac/v1"
1719
"k8s.io/apimachinery/pkg/api/equality"
1820
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1921
kubeinformers "k8s.io/client-go/informers"
2022
"k8s.io/client-go/kubernetes"
2123
fakekube "k8s.io/client-go/kubernetes/fake"
24+
certutil "k8s.io/client-go/util/cert"
2225
"k8s.io/klog/v2/ktesting"
2326

2427
addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
@@ -248,11 +251,28 @@ func TestTemplateCSRApproveCheckFunc(t *testing.T) {
248251
}
249252

250253
func TestTemplateCSRSignFunc(t *testing.T) {
254+
ca, key, err := certutil.GenerateSelfSignedCertKey("test", []net.IP{}, []string{})
255+
if err != nil {
256+
t.Errorf("Failed to generate self signed CA config: %v", err)
257+
}
258+
secret := &corev1.Secret{
259+
ObjectMeta: metav1.ObjectMeta{
260+
Name: "test",
261+
Namespace: "test-ns",
262+
},
263+
Data: map[string][]byte{
264+
corev1.TLSCertKey: ca,
265+
corev1.TLSPrivateKeyKey: key,
266+
},
267+
Type: corev1.SecretTypeTLS,
268+
}
269+
251270
cases := []struct {
252271
name string
253272
cluster *clusterv1.ManagedCluster
254273
addon *addonapiv1alpha1.ManagedClusterAddOn
255274
template *addonapiv1alpha1.AddOnTemplate
275+
casecret *corev1.Secret
256276
csr *certificatesv1.CertificateSigningRequest
257277
expectedCert []byte
258278
}{
@@ -325,11 +345,53 @@ func TestTemplateCSRSignFunc(t *testing.T) {
325345
},
326346
expectedCert: nil,
327347
},
348+
{
349+
name: "customsigner with ca secret",
350+
casecret: secret,
351+
cluster: NewFakeManagedCluster("cluster1"),
352+
template: NewFakeAddonTemplate("template1", []addonapiv1alpha1.RegistrationSpec{
353+
{
354+
Type: addonapiv1alpha1.RegistrationTypeCustomSigner,
355+
CustomSigner: &addonapiv1alpha1.CustomSignerRegistrationConfig{
356+
SignerName: "s1",
357+
Subject: &addonapiv1alpha1.Subject{
358+
User: "u1",
359+
Groups: []string{
360+
"g1",
361+
"g2",
362+
},
363+
OrganizationUnits: []string{},
364+
},
365+
SigningCA: addonapiv1alpha1.SigningCARef{
366+
Name: secret.Name,
367+
Namespace: secret.Namespace,
368+
},
369+
},
370+
},
371+
}),
372+
addon: NewFakeTemplateManagedClusterAddon("addon1", "cluster1", "template1", "fakehash"),
373+
csr: &certificatesv1.CertificateSigningRequest{
374+
ObjectMeta: metav1.ObjectMeta{
375+
Name: "csr1",
376+
Labels: map[string]string{
377+
clusterv1.ClusterNameLabelKey: "cluster1",
378+
},
379+
},
380+
Spec: certificatesv1.CertificateSigningRequestSpec{
381+
SignerName: "s1",
382+
Username: "system:open-cluster-management:cluster1:adcde",
383+
},
384+
},
385+
expectedCert: nil,
386+
},
328387
}
329388
for _, c := range cases {
330389
_, ctx := ktesting.NewTestContext(t)
331390
addonClient := fakeaddon.NewSimpleClientset(c.template, c.addon)
332391
hubKubeClient := fakekube.NewSimpleClientset()
392+
if c.casecret != nil {
393+
hubKubeClient = fakekube.NewSimpleClientset(c.casecret)
394+
}
333395
addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, 30*time.Minute)
334396
mcaStore := addonInformerFactory.Addon().V1alpha1().ManagedClusterAddOns().Informer().GetStore()
335397
if err := mcaStore.Add(c.addon); err != nil {

‎test/e2e/addonmanagement_test.go

+37-15
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,24 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered,
7171
"addon/signca_secret_rolebinding.yaml",
7272
}
7373

74+
var signerSecretNamespace string
75+
7476
ginkgo.BeforeEach(func() {
77+
signerSecretNamespace = "signer-secret-test-ns" + rand.String(6)
78+
79+
ginkgo.By("create addon custom sign secret namespace")
80+
_, err := hub.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &corev1.Namespace{
81+
ObjectMeta: metav1.ObjectMeta{
82+
Name: signerSecretNamespace,
83+
},
84+
}, metav1.CreateOptions{})
85+
if err != nil && !errors.IsAlreadyExists(err) {
86+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
87+
}
88+
7589
ginkgo.By("create addon custom sign secret")
76-
err := copySignerSecret(context.TODO(), hub.KubeClient, "open-cluster-management-hub",
77-
"signer-secret", templateagent.AddonManagerNamespace(), customSignerSecretName)
90+
err = copySignerSecret(context.TODO(), hub.KubeClient, "open-cluster-management-hub",
91+
"signer-secret", signerSecretNamespace, customSignerSecretName)
7892
gomega.Expect(err).ToNot(gomega.HaveOccurred())
7993

8094
// the addon manager deployment should be running
@@ -85,11 +99,12 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered,
8599
ginkgo.By(fmt.Sprintf("create addon template resources for cluster %v", universalClusterName))
86100
err = createResourcesFromYamlFiles(context.Background(), hub.DynamicClient, hub.RestMapper, s,
87101
defaultAddonTemplateReaderManifestsFunc(manifests.AddonManifestFiles, map[string]interface{}{
88-
"Namespace": universalClusterName,
89-
"AddonInstallNamespace": addonInstallNamespace,
90-
"CustomSignerName": customSignerName,
91-
"AddonManagerNamespace": templateagent.AddonManagerNamespace(),
92-
"CustomSignerSecretName": customSignerSecretName,
102+
"Namespace": universalClusterName,
103+
"AddonInstallNamespace": addonInstallNamespace,
104+
"CustomSignerName": customSignerName,
105+
"AddonManagerNamespace": templateagent.AddonManagerNamespace(),
106+
"CustomSignerSecretName": customSignerSecretName,
107+
"CustomSignerSecretNamespace": signerSecretNamespace,
93108
}),
94109
templateResources,
95110
)
@@ -132,22 +147,29 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered,
132147
ginkgo.By(fmt.Sprintf("delete addon template resources for cluster %v", universalClusterName))
133148
err = deleteResourcesFromYamlFiles(context.Background(), hub.DynamicClient, hub.RestMapper, s,
134149
defaultAddonTemplateReaderManifestsFunc(manifests.AddonManifestFiles, map[string]interface{}{
135-
"Namespace": universalClusterName,
136-
"AddonInstallNamespace": addonInstallNamespace,
137-
"CustomSignerName": customSignerName,
138-
"AddonManagerNamespace": templateagent.AddonManagerNamespace(),
139-
"CustomSignerSecretName": customSignerSecretName,
150+
"Namespace": universalClusterName,
151+
"AddonInstallNamespace": addonInstallNamespace,
152+
"CustomSignerName": customSignerName,
153+
"AddonManagerNamespace": templateagent.AddonManagerNamespace(),
154+
"CustomSignerSecretName": customSignerSecretName,
155+
"CustomSignerSecretNamespace": signerSecretNamespace,
140156
}),
141157
templateResources,
142158
)
143159
gomega.Expect(err).ToNot(gomega.HaveOccurred())
144160

145161
ginkgo.By("delete addon custom sign secret")
146-
err = hub.KubeClient.CoreV1().Secrets(templateagent.AddonManagerNamespace()).Delete(context.TODO(),
162+
err = hub.KubeClient.CoreV1().Secrets(signerSecretNamespace).Delete(context.TODO(),
147163
customSignerSecretName, metav1.DeleteOptions{})
148164
if err != nil && !errors.IsNotFound(err) {
149165
ginkgo.Fail(fmt.Sprintf("failed to delete custom signer secret %v/%v: %v",
150-
templateagent.AddonManagerNamespace(), customSignerSecretName, err))
166+
signerSecretNamespace, customSignerSecretName, err))
167+
}
168+
169+
ginkgo.By("delete addon custom sign secret namespace")
170+
err = hub.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), signerSecretNamespace, metav1.DeleteOptions{})
171+
if err != nil && !errors.IsNotFound(err) {
172+
ginkgo.Fail(fmt.Sprintf("failed to delete custom signer secret namespace %v: %v", signerSecretNamespace, err))
151173
}
152174

153175
// delete all CSR created for the addon on the hub cluster, otherwise if it reches the limit number 10, the
@@ -182,7 +204,7 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered,
182204
return err
183205
}).Should(gomega.Succeed())
184206

185-
ginkgo.By("Check custom signer secret is created")
207+
ginkgo.By("Check custom client cert secret is created")
186208
gomega.Eventually(func() error {
187209
_, err := hub.KubeClient.CoreV1().Secrets(addonInstallNamespace).Get(context.TODO(),
188210
templateagent.CustomSignedSecretName(addOnName, customSignerName), metav1.GetOptions{})

‎test/e2e/manifests/addon/addon_template.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@ spec:
152152
signerName: << CustomSignerName >>
153153
signingCA:
154154
name: << CustomSignerSecretName >>
155+
namespace: << CustomSignerSecretNamespace >>
155156
subject:
156157
groups:
157158
- g1

‎test/e2e/manifests/addon/signca_secret_role.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1
22
kind: Role
33
metadata:
44
name: get-customer-ca
5-
namespace: << AddonManagerNamespace >>
5+
namespace: << CustomSignerSecretNamespace >>
66
rules:
77
- apiGroups:
88
- ""

‎test/e2e/manifests/addon/signca_secret_rolebinding.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1
33
kind: RoleBinding
44
metadata:
55
name: get-customer-ca
6-
namespace: << AddonManagerNamespace >>
6+
namespace: << CustomSignerSecretNamespace >>
77
roleRef:
88
apiGroup: rbac.authorization.k8s.io
99
kind: Role

‎vendor/modules.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1584,7 +1584,7 @@ open-cluster-management.io/addon-framework/pkg/agent
15841584
open-cluster-management.io/addon-framework/pkg/assets
15851585
open-cluster-management.io/addon-framework/pkg/index
15861586
open-cluster-management.io/addon-framework/pkg/utils
1587-
# open-cluster-management.io/api v0.15.1-0.20241126073717-05ff7c1affe8
1587+
# open-cluster-management.io/api v0.15.1-0.20241209025232-b62746ae96d4
15881588
## explicit; go 1.22.0
15891589
open-cluster-management.io/api/addon/v1alpha1
15901590
open-cluster-management.io/api/client/addon/clientset/versioned

‎vendor/open-cluster-management.io/api/addon/v1alpha1/0000_03_addon.open-cluster-management.io_addontemplates.crd.yaml

+8-4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/open-cluster-management.io/api/addon/v1alpha1/types_addontemplate.go

+6-4
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vendor/open-cluster-management.io/api/addon/v1alpha1/zz_generated.swagger_doc_generated.go

+4-3
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)