Skip to content

Commit

Permalink
Add admission controller (#513)
Browse files Browse the repository at this point in the history
* Add CSR creation logic

* Added BootstrapAuthenticator and CSR registration implementation

CSR registration wnd BootstrapAuthenticator will be handled behind SecureAccess feature flag.
CSR resource will be created in the management cluster on the start of
the HostAgent.
BootstrapAuthenticator controller will watch the CSR resources.
Currently placeholders are created for different flows and the
implementation will be raised in differemt PRs

* Changes as per review comments

* Added Admission Coontroller

* Added tests and added contoller in main

Signed-off-by: Sachin Kumar Singh <[email protected]>

* Added RBAC permissions for the controller

Signed-off-by: Sachin Kumar Singh <[email protected]>

* Add certificatev1 scheme in e2e tests

Signed-off-by: Sachin Kumar Singh <[email protected]>

* Add list permissions to certs and refactor main

Signed-off-by: Sachin Kumar Singh <[email protected]>

* improved tests

Signed-off-by: Sachin Kumar Singh <[email protected]>

* fix lints

Signed-off-by: Sachin Kumar Singh <[email protected]>

* rebase fixes in host_agent_test.go

Signed-off-by: Sachin Kumar Singh <[email protected]>

* review changes

Signed-off-by: Sachin Kumar Singh <[email protected]>

* remove unwanted autogenerated permissions

Signed-off-by: Sachin Kumar Singh <[email protected]>

* refactored CSR condition check

Signed-off-by: Sachin Kumar Singh <[email protected]>

Co-authored-by: Dharmjit Singh <[email protected]>
  • Loading branch information
sachinkumarsingh092 and Dharmjit Singh committed May 12, 2022
1 parent ca6f3e9 commit c917aec
Show file tree
Hide file tree
Showing 6 changed files with 241 additions and 24 deletions.
1 change: 1 addition & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ rules:
verbs:
- create
- get
- list
- watch
- apiGroups:
- cluster.x-k8s.io
Expand Down
98 changes: 98 additions & 0 deletions controllers/infrastructure/byoadmission_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
// Copyright 2022 VMware, Inc. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package controllers

import (
"context"
"strings"

certv1 "k8s.io/api/certificates/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

// ByoAdmissionReconciler reconciles a ByoAdmission object
type ByoAdmissionReconciler struct {
ClientSet clientset.Interface
}

//+kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=create;get;list;watch

// Reconcile continuosuly checks for CSRs and approves them
func (r *ByoAdmissionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
var err error
logger := log.FromContext(ctx)
logger.Info("Reconcile request received", "object", req.NamespacedName)

// Fetch the CSR from the api-server
csr, err := r.ClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, req.NamespacedName.Name, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
logger.Error(err, "CertificateSigningRequest not found, won't reconcile")
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
}

// Check if the CSR is already approved or denied
csrApproved := checkCSRCondition(csr.Status.Conditions, certv1.CertificateApproved)
csrDenied := checkCSRCondition(csr.Status.Conditions, certv1.CertificateDenied)
if csrApproved || csrDenied {
if csrApproved {
logger.Info("CertificateSigningRequest is already approved", "CSR", csr.Name)
}
if csrDenied {
logger.Info("CertificateSigningRequest is already denied", "CSR", csr.Name)
}
return ctrl.Result{}, nil
}

// Update the CSR to the "Approved" condition
csr.Status.Conditions = append(csr.Status.Conditions, certv1.CertificateSigningRequestCondition{
Type: certv1.CertificateApproved,
Reason: "Approved by ByoAdmission Controller",
})

// Approve the CSR
logger.Info("Approving CSR", "object", req.NamespacedName)
_, err = r.ClientSet.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{})
if err != nil {
return reconcile.Result{}, err
}

logger.Info("CSR Approved", "object", req.NamespacedName)

return ctrl.Result{}, nil
}

// Check if the CSR has the given condition.
func checkCSRCondition(conditions []certv1.CertificateSigningRequestCondition, conditionType certv1.RequestConditionType) bool {
for _, condition := range conditions {
if condition.Type == conditionType {
return true
}
}
return false
}

// SetupWithManager sets up the controller with the Manager.
func (r *ByoAdmissionReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&certv1.CertificateSigningRequest{}).WithEventFilter(
// watch only BYOH created CSRs
predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool {
return strings.HasPrefix(e.Object.GetName(), "byoh-csr-")
},
UpdateFunc: func(e event.UpdateEvent) bool {
return strings.HasPrefix(e.ObjectOld.GetName(), "byoh-csr-")
}}).
Complete(r)
}
97 changes: 97 additions & 0 deletions controllers/infrastructure/byoadmission_controller_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
// Copyright 2021 VMware, Inc. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package controllers_test

import (
"context"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/test/builder"
certv1 "k8s.io/api/certificates/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

var _ = Describe("Controllers/ByoadmissionController", func() {
var (
ctx context.Context
err error
CSR *certv1.CertificateSigningRequest
)

It("should return error for non-existent CSR", func() {
// Call Reconcile method for a non-existing CSR
objectKey := types.NamespacedName{Name: defaultByoHostName}
_, err = byoAdmissionReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objectKey})
Expect(err).To(BeNil())
})

Context("When a CSR is created", func() {
BeforeEach(func() {
ctx = context.Background()

// Create a CSR resource for each test
CSR, err = builder.CertificateSigningRequest(defaultByoHostName, "test-cn", "test-org", 2048).Build()
Expect(err).NotTo(HaveOccurred())
})

It("should approve the Byoh CSR", func() {
// Create a dummy CSR request
_, err = clientSetFake.CertificatesV1().CertificateSigningRequests().Create(ctx, CSR, v1.CreateOptions{})
Expect(err).ToNot(HaveOccurred())

// Call Reconcile method
objectKey := types.NamespacedName{Name: defaultByoHostName}
_, err = byoAdmissionReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objectKey})
Expect(err).ShouldNot(HaveOccurred())

// Fetch the updated CSR
var updateByohCSR *certv1.CertificateSigningRequest
updateByohCSR, err = clientSetFake.CertificatesV1().CertificateSigningRequests().Get(ctx, defaultByoHostName, v1.GetOptions{})
Expect(err).ToNot(HaveOccurred())
Expect(updateByohCSR.Status.Conditions).Should(ContainElement(certv1.CertificateSigningRequestCondition{
Type: certv1.CertificateApproved,
Reason: "Approved by ByoAdmission Controller",
}))
})

It("should not approve a denied CSR", func() {
// Create a fake denied CSR request
CSR.Status.Conditions = append(CSR.Status.Conditions, certv1.CertificateSigningRequestCondition{
Type: certv1.CertificateDenied,
})

_, err = clientSetFake.CertificatesV1().CertificateSigningRequests().Create(ctx, CSR, v1.CreateOptions{})
Expect(err).ToNot(HaveOccurred())

// Call Reconcile method
objectKey := types.NamespacedName{Name: defaultByoHostName}
_, err = byoAdmissionReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objectKey})
Expect(err).To(BeNil())
})

It("should not approve an already approved CSR", func() {
// Create a fake approved CSR request
CSR.Status.Conditions = append(CSR.Status.Conditions, certv1.CertificateSigningRequestCondition{
Type: certv1.CertificateApproved,
})

_, err = clientSetFake.CertificatesV1().CertificateSigningRequests().Create(ctx, CSR, v1.CreateOptions{})
Expect(err).ToNot(HaveOccurred())

// Call Reconcile method
objectKey := types.NamespacedName{Name: defaultByoHostName}
_, err = byoAdmissionReconciler.Reconcile(ctx, reconcile.Request{NamespacedName: objectKey})
Expect(err).To(BeNil())
})

AfterEach(func() {
Expect(clientSetFake.CertificatesV1().CertificateSigningRequests().Delete(ctx, defaultByoHostName, v1.DeleteOptions{})).ShouldNot(HaveOccurred())
})

})

})
45 changes: 27 additions & 18 deletions controllers/infrastructure/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (

//+kubebuilder:scaffold:imports

fakeclientset "k8s.io/client-go/kubernetes/fake"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
"sigs.k8s.io/cluster-api/controllers/remote"
Expand All @@ -38,24 +39,26 @@ import (
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.

var (
testEnv *envtest.Environment
clientFake client.Client
reconciler *controllers.ByoMachineReconciler
byoClusterReconciler *controllers.ByoClusterReconciler
recorder *record.FakeRecorder
byoCluster *infrastructurev1beta1.ByoCluster
capiCluster *clusterv1.Cluster
defaultClusterName = "my-cluster"
defaultNodeName = "my-host"
defaultByoHostName = "my-host"
defaultMachineName = "my-machine"
defaultByoMachineName = "my-byomachine"
defaultNamespace = "default"
fakeBootstrapSecret = "fakeBootstrapSecret"
k8sManager ctrl.Manager
cfg *rest.Config
ctx context.Context
cancel context.CancelFunc
testEnv *envtest.Environment
clientFake client.Client
clientSetFake = fakeclientset.NewSimpleClientset()
reconciler *controllers.ByoMachineReconciler
byoClusterReconciler *controllers.ByoClusterReconciler
byoAdmissionReconciler *controllers.ByoAdmissionReconciler
recorder *record.FakeRecorder
byoCluster *infrastructurev1beta1.ByoCluster
capiCluster *clusterv1.Cluster
defaultClusterName = "my-cluster"
defaultNodeName = "my-host"
defaultByoHostName = "my-host"
defaultMachineName = "my-machine"
defaultByoMachineName = "my-byomachine"
defaultNamespace = "default"
fakeBootstrapSecret = "fakeBootstrapSecret"
k8sManager ctrl.Manager
cfg *rest.Config
ctx context.Context
cancel context.CancelFunc
)

func TestAPIs(t *testing.T) {
Expand Down Expand Up @@ -132,6 +135,12 @@ var _ = BeforeSuite(func() {
err = byoClusterReconciler.SetupWithManager(k8sManager)
Expect(err).NotTo(HaveOccurred())

byoAdmissionReconciler = &controllers.ByoAdmissionReconciler{
ClientSet: clientSetFake,
}
err = byoAdmissionReconciler.SetupWithManager(k8sManager)
Expect(err).NotTo(HaveOccurred())

go func() {
err = k8sManager.GetCache().Start(ctx)
Expect(err).NotTo(HaveOccurred())
Expand Down
22 changes: 16 additions & 6 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
admissionv1beta1 "k8s.io/api/admission/v1beta1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientset "k8s.io/client-go/kubernetes"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/healthz"
Expand All @@ -32,8 +33,11 @@ import (
)

var (
scheme = runtime.NewScheme()
setupLog = ctrl.Log.WithName("setup")
scheme = runtime.NewScheme()
setupLog = ctrl.Log.WithName("setup")
metricsAddr string
enableLeaderElection bool
probeAddr string
)

func init() {
Expand All @@ -47,16 +51,16 @@ func init() {
utilruntime.Must(admissionv1beta1.AddToScheme(scheme))
}

func main() {
var metricsAddr string
var enableLeaderElection bool
var probeAddr string
func setFlags() {
flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flag.Parse()
}

func main() {
setFlags()
ctrl.SetLogger(klogr.New())

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Expand Down Expand Up @@ -118,6 +122,12 @@ func main() {
setupLog.Error(err, "unable to create controller", "controller", "ByoCluster")
os.Exit(1)
}
if err = (&byohcontrollers.ByoAdmissionReconciler{
ClientSet: clientset.NewForConfigOrDie(ctrl.GetConfigOrDie()),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ByoAdmission")
os.Exit(1)
}

if err = (&infrastructurev1beta1.ByoCluster{}).SetupWebhookWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create webhook", "webhook", "ByoCluster")
Expand Down
2 changes: 2 additions & 0 deletions test/e2e/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gexec"
infraproviderv1 "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/apis/infrastructure/v1beta1"
certv1 "k8s.io/api/certificates/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
Expand Down Expand Up @@ -151,6 +152,7 @@ var _ = SynchronizedAfterSuite(func() {
func initScheme() *runtime.Scheme {
sc := runtime.NewScheme()
framework.TryAddDefaultSchemes(sc)
Expect(certv1.AddToScheme(sc)).NotTo(HaveOccurred())
Expect(infraproviderv1.AddToScheme(sc)).NotTo(HaveOccurred())
return sc
}
Expand Down

0 comments on commit c917aec

Please sign in to comment.