feat(pod-template): add tor service pod template (#18)

* feat(pod-template): add tor service pod template

* modify v1alpha2 instead of creating v1alpha3

* use corev1.PodSpec and add balancerTemplate to OnionBalancedService

* minor fixes from PR comments

* minor fix to readme text

Makefile

@@ -45,6 +45,7 @@ help: ## Display this help.
 .PHONY: manifests
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	hack/remove-containers-requirement.sh config/crd/bases
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -108,7 +109,7 @@ endif
 
 .PHONY: install
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+	$(KUSTOMIZE) build config/crd | kubectl replace --force -f -
 
 .PHONY: uninstall
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.

README.md

@@ -21,6 +21,7 @@
    1. [Quickstart with random address](#quickstart-with-random-address)
    1. [Random service names](#random-service-names)
    1. [Bring your own secret](#bring-your-own-secret)
+   1. [Specify pod template settings](#specify-pod-template-settings)
    1. [Using with nginx-ingress](#using-with-nginx-ingress)
    1. [HA Onionbalance Hidden Services](#ha-onionbalance-hidden-services)
    1. [Service Monitors](#service-monitors)
@@ -221,6 +222,94 @@ Only v3 is supported.
 
 tor-controller defaults to using v3 if `spec.version` is not specified.
 
+Specify Pod Template Settings
+-----------------------------
+
+The `template` field can be used to specify properties for the running tor-service pods.
+Use `template.resources` to specify the compute resources required by the tor containers that will be created.
+
+```yaml
+apiVersion: tor.k8s.torproject.org/v1alpha2
+kind: OnionService
+metadata:
+  name: example-onion-service
+spec:
+  ...
+  template:
+    metadata:
+      annotations:
+        some-special-anotation: my-value
+    spec:
+      # nodeSelector:
+      # affinity:
+      # schedulerName:
+      # tolerations:
+      # priorityClassName:
+      # runtimeClassName:
+      # topologySpreadConstraints:
+    resources:
+      limits:
+        cpu: 500m
+        memory: 128Mi
+```
+
+| Template Property | Description |
+| -------- | ----------- |
+| `metadata.annotations` | Add pod [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/). |
+| `metadata.labels` | Add pod [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/). NOTE: `tor-controller` automatically adds the labels `app` and `controller`, so you should not set these labels |
+| `spec.nodeSelector` | Set specific [Node Selectors](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the pod. |
+| `spec.affinity` | Add pod or node [affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) rules here. |
+| `spec.schedulerName` | Select a specific [scheduler](https://kubernetes.io/docs/concepts/scheduling-eviction/kube-scheduler/) to be used for service pods |
+| `spec.tolerations` | Add [tolerations](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#toleration-v1-core) to the pods. |
+| `spec.runtimeClassName` | Set the pods [Runtime Class](https://kubernetes.io/docs/concepts/containers/runtime-class/). |
+| `spec.priorityClassName` | Set the pods [Priority Class](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass) |
+| `spec.resources` | Set [Resource Requirements](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container) for the running containers. |
+| `spec.topologySpreadConstraints` | Add [Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). |
+| `resources` | Set [Resource Requirements](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container) for the running containers. |
+
+
+OnionBalancedService Pod Template
+---------------------------------
+
+In addition to creating backend `OnionServices`, a OnionBalancedService also creates a deployment that runs the Onion Balancer.  To modify the pod settings for the balancer service, you can specify the a `balancerTemplate` property in the `OnionBalancedServie` spec.
+
+```yaml
+apiVersion: tor.k8s.torproject.org/v1alpha2
+kind: OnionBalancedService
+metadata:
+  name: example-onion-service
+spec:
+  ...
+  balancerTemplate:
+    spec:
+      # nodeSelector:
+      # affinity:
+      # schedulerName:
+      # tolerations:
+      # priorityClassName:
+      # runtimeClassName:
+```
+
+Additionally, the Onion Balancer pod contains two separate containers, which can each have their resource requirements set via `balancerTemplate.torResources` and `balancerTemplate.balancerResources`.
+
+```yaml
+apiVersion: tor.k8s.torproject.org/v1alpha2
+kind: OnionBalancedService
+metadata:
+  name: example-onion-service
+spec:
+  ...
+  balancerTemplate:
+    torResources:
+      limits:
+        cpu: 500m
+        memory: 128Mi
+    balancerResources:
+      limits:
+        cpu: 500m
+        memory: 128Mi
+```
+
 
 Using with nginx-ingress
 ------------------------

agents/onionbalance/local/client.go

@@ -28,7 +28,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	onionbalancedaemon "github.com/bugfest/tor-controller/agents/onionbalance/onionbalancedaemon"
-
 	torv1alpha2 "github.com/bugfest/tor-controller/apis/tor/v1alpha2"
 )
 

agents/tor/local/client.go

@@ -28,7 +28,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	tordaemon "github.com/bugfest/tor-controller/agents/tor/tordaemon"
-
 	torv1alpha2 "github.com/bugfest/tor-controller/apis/tor/v1alpha2"
 )
 

apis/tor/v1alpha2/onionbalancedservice_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha2
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -47,13 +48,36 @@ type OnionBalancedServiceSpec struct {
 
 	// +optional
 	Template TemplateReference `json:"template,omitempty"`
+
+	// Template describes the balancer daemon pods that will be created.
+	// +optional
+	BalancerTemplate BalancerTemplate `json:"balancerTemplate,omitempty"`
 }
 
 type TemplateReference struct {
 	// +optional
 	Spec OnionServiceSpec `json:"spec,omitempty"`
 }
 
+// Template for the daemon pods
+type BalancerTemplate struct {
+	// Metadata of the pods created from this template.
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Spec defines the behavior of a pod.
+	// +optional
+	Spec corev1.PodSpec `json:"spec,omitempty"`
+
+	// Default resources for tor containers
+	// +optional
+	TorResources corev1.ResourceRequirements `json:"torResources,omitempty" protobuf:"bytes,8,opt,name=resources"`
+
+	// Default resources for onionbalance containers
+	// +optional
+	BalancerResources corev1.ResourceRequirements `json:"balancerResources,omitempty" protobuf:"bytes,8,opt,name=resources"`
+}
+
 // OnionBalancedServiceStatus defines the observed state of OnionBalancedService
 type OnionBalancedServiceStatus struct {
 	// +optional

apis/tor/v1alpha2/onionbalancedservice_util.go

@@ -1,6 +1,10 @@
 package v1alpha2
 
-import "fmt"
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+)
 
 const (
 	onionbalanceDeploymentNameFmt     = "%s-tor-daemon"
@@ -87,3 +91,18 @@ func (s *OnionBalancedService) IsSynced() bool {
 	}
 	return true
 }
+
+func (s *OnionBalancedService) PodTemplate() corev1.PodTemplateSpec {
+	return corev1.PodTemplateSpec{
+		ObjectMeta: s.Spec.BalancerTemplate.ObjectMeta,
+		Spec:       s.Spec.BalancerTemplate.Spec,
+	}
+}
+
+func (s *OnionBalancedService) TorResources() corev1.ResourceRequirements {
+	return s.Spec.BalancerTemplate.TorResources
+}
+
+func (s *OnionBalancedService) BalancerResources() corev1.ResourceRequirements {
+	return s.Spec.BalancerTemplate.BalancerResources
+}

apis/tor/v1alpha2/onionservice_types.go

@@ -17,19 +17,38 @@ limitations under the License.
 package v1alpha2
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
+type ServicePodTemplate struct {
+	// Metadata of the pods created from this template.
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Spec defines the behavior of a pod.
+	// +optional
+	Spec corev1.PodSpec `json:"spec,omitempty"`
+
+	// Default resources for containers
+	// +optional
+	Resources corev1.ResourceRequirements `json:"resources,omitempty" protobuf:"bytes,8,opt,name=resources"`
+}
+
 // OnionServiceSpec defines the desired state of OnionService
 type OnionServiceSpec struct {
 	// +patchMergeKey=port
 	// +patchStrategy=merge
 	Rules []ServiceRule `json:"rules,omitempty" pathchStrategy:"merge" patchMergeKey:"port"`
 
+	// Template describes the pods that will be created.
+	// +optional
+	Template ServicePodTemplate `json:"template,omitempty"`
+
 	// +optional
 	PrivateKeySecret SecretReference `json:"privateKeySecret,omitempty"`
 

apis/tor/v1alpha2/onionservice_util.go

@@ -1,6 +1,10 @@
 package v1alpha2
 
-import "fmt"
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+)
 
 const (
 	torDeploymentNameFmt       = "%s-tor-daemon"
@@ -69,3 +73,14 @@ func (s *OnionService) RoleName() string {
 func (s *OnionService) ServiceAccountName() string {
 	return fmt.Sprintf(torServiceAccountNameFmt, s.Name)
 }
+
+func (s *OnionService) PodTemplate() corev1.PodTemplateSpec {
+	return corev1.PodTemplateSpec{
+		ObjectMeta: s.Spec.Template.ObjectMeta,
+		Spec:       s.Spec.Template.Spec,
+	}
+}
+
+func (s *OnionService) Resources() corev1.ResourceRequirements {
+	return s.Spec.Template.Resources
+}

config/crd/bases/config.k8s.torproject.org_projectconfigs.yaml

@@ -189,4 +189,4 @@ status:
     plural: ""
   conditions: []
   storedVersions: []
-
+