Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions api/hypershift/v1beta1/hostedcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,11 @@ const (
// It is not set by the end-user.
DisableClusterAutoscalerAnnotation = "hypershift.openshift.io/disable-cluster-autoscaler"

// DisableAWSNodeTerminationHandlerAnnotation allows disabling the AWS Node Termination Handler for a hosted cluster.
// This annotation is only set by the hypershift-operator on HostedControlPlanes.
// It is not set by the end-user.
DisableAWSNodeTerminationHandlerAnnotation = "hypershift.openshift.io/disable-aws-node-termination-handler"

// AroHCP represents the ARO HCP managed service offering
AroHCP = "ARO-HCP"

Expand Down
24 changes: 24 additions & 0 deletions cmd/infra/aws/delegating_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"github.com/aws/aws-sdk-go/service/route53/route53iface"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/aws/aws-sdk-go/service/sqs"
"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
)

// NewDelegatingClient creates a new set of AWS service clients that delegate individual calls to the right credentials.
Expand Down Expand Up @@ -87,6 +89,7 @@ func NewDelegatingClient(
})
nodePool := &nodePoolClientDelegate{
ec2Client: ec2.New(nodePoolSession, awsConfig),
sqsClient: sqs.New(nodePoolSession, awsConfig),
}
openshiftImageRegistrySession, err := session.NewSessionWithOptions(session.Options{SharedConfigFiles: []string{openshiftImageRegistryCredentialsFile}})
if err != nil {
Expand Down Expand Up @@ -124,6 +127,10 @@ func NewDelegatingClient(
S3API: nil,
openshiftImageRegistry: openshiftImageRegistry,
},
SQSAPI: &sqsClient{
SQSAPI: nil,
nodePool: nodePool,
},
}, nil
}

Expand All @@ -148,6 +155,7 @@ type controlPlaneOperatorClientDelegate struct {

type nodePoolClientDelegate struct {
ec2Client ec2iface.EC2API
sqsClient sqsiface.SQSAPI
}

type openshiftImageRegistryClientDelegate struct {
Expand All @@ -161,6 +169,7 @@ type DelegatingClient struct {
elbv2iface.ELBV2API
route53iface.Route53API
s3iface.S3API
sqsiface.SQSAPI
}

// ec2Client delegates to individual component clients for API calls we know those components will have privileges to make.
Expand Down Expand Up @@ -593,3 +602,18 @@ func (c *s3Client) PutObjectWithContext(ctx aws.Context, input *s3.PutObjectInpu
func (c *s3Client) PutPublicAccessBlockWithContext(ctx aws.Context, input *s3.PutPublicAccessBlockInput, opts ...request.Option) (*s3.PutPublicAccessBlockOutput, error) {
return c.openshiftImageRegistry.s3Client.PutPublicAccessBlockWithContext(ctx, input, opts...)
}

// sqsClient delegates to individual component clients for API calls we know those components will have privileges to make.
type sqsClient struct {
// embedding this fulfills the interface and falls back to a panic for APIs we don't have privileges for
sqsiface.SQSAPI

nodePool *nodePoolClientDelegate
}

func (c *sqsClient) DeleteMessageWithContext(ctx aws.Context, input *sqs.DeleteMessageInput, opts ...request.Option) (*sqs.DeleteMessageOutput, error) {
return c.nodePool.sqsClient.DeleteMessageWithContext(ctx, input, opts...)
}
func (c *sqsClient) ReceiveMessageWithContext(ctx aws.Context, input *sqs.ReceiveMessageInput, opts ...request.Option) (*sqs.ReceiveMessageOutput, error) {
return c.nodePool.sqsClient.ReceiveMessageWithContext(ctx, input, opts...)
}
4 changes: 3 additions & 1 deletion cmd/infra/aws/iam.go
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,9 @@ var (
"ec2:DescribeLaunchTemplates",
"ec2:DescribeLaunchTemplateVersions",
"ec2:DeleteLaunchTemplate",
"ec2:DeleteLaunchTemplateVersions"
"ec2:DeleteLaunchTemplateVersions",
"sqs:DeleteMessage",
"sqs:ReceiveMessage"
],
"Resource": [
"*"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/oauth"
"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/pki"
autoscalerv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/autoscaler"
awsnodeterminationhandlerv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/awsnodeterminationhandler"
awsccmv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/aws"
azureccmv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/azure"
kubevirtccmv2 "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/kubevirt"
Expand Down Expand Up @@ -255,6 +256,7 @@ func (r *HostedControlPlaneReconciler) registerComponents(hcp *hyperv1.HostedCon
ntov2.NewComponent(),
dnsoperatorv2.NewComponent(),
machineapproverv2.NewComponent(),
awsnodeterminationhandlerv2.NewComponent(),
ingressoperatorv2.NewComponent(),
snapshotcontrollerv2.NewComponent(),
registryoperatorv2.NewComponent(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1820,8 +1820,12 @@ func TestControlPlaneComponents(t *testing.T) {
Labels: map[string]string{
"cluster.x-k8s.io/cluster-name": "cluster_name",
},
Annotations: map[string]string{
"hypershift.openshift.io/aws-termination-handler-queue-url": "https://sqs.us-east-1.amazonaws.com/123456789012/test-queue",
},
},
Spec: hyperv1.HostedControlPlaneSpec{
IssuerURL: "https://test-oidc-bucket.s3.us-east-1.amazonaws.com/test-cluster",
Configuration: &hyperv1.ClusterConfiguration{
FeatureGate: &configv1.FeatureGateSpec{},
},
Expand All @@ -1845,7 +1849,11 @@ func TestControlPlaneComponents(t *testing.T) {
},
Platform: hyperv1.PlatformSpec{
Type: hyperv1.AWSPlatform,
AWS: &hyperv1.AWSPlatformSpec{},
AWS: &hyperv1.AWSPlatformSpec{
RolesRef: hyperv1.AWSRolesRef{
NodePoolManagementARN: "arn:aws:iam::123456789012:role/test-node-pool-management-role",
},
},
Azure: &hyperv1.AzurePlatformSpec{
SubnetID: "/subscriptions/mySubscriptionID/resourceGroups/myResourceGroupName/providers/Microsoft.Network/virtualNetworks/myVnetName/subnets/mySubnetName",
SecurityGroupID: "/subscriptions/mySubscriptionID/resourceGroups/myResourceGroupName/providers/Microsoft.Network/networkSecurityGroups/myNSGName",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
apiVersion: hypershift.openshift.io/v1beta1
kind: ControlPlaneComponent
metadata:
name: aws-node-termination-handler
namespace: hcp-namespace
resourceVersion: "1"
spec: {}
status:
conditions:
- lastTransitionTime: null
message: aws-node-termination-handler Deployment Available condition not found
reason: NotFound
status: "False"
type: Available
- lastTransitionTime: null
message: 'Waiting for deployment aws-node-termination-handler rollout to finish:
0 out of 1 new replicas have been updated'
reason: WaitingForRolloutComplete
status: "False"
type: RolloutComplete
resources:
- group: ""
kind: Secret
name: aws-node-termination-handler-creds
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: v1
data:
credentials: W2RlZmF1bHRdCnJvbGVfYXJuID0gYXJuOmF3czppYW06OjEyMzQ1Njc4OTAxMjpyb2xlL3Rlc3Qtbm9kZS1wb29sLW1hbmFnZW1lbnQtcm9sZQp3ZWJfaWRlbnRpdHlfdG9rZW5fZmlsZSA9IC92YXIvcnVuL3NlY3JldHMvb3BlbnNoaWZ0L3NlcnZpY2VhY2NvdW50L3Rva2VuCnN0c19yZWdpb25hbF9lbmRwb2ludHMgPSByZWdpb25hbAo=
kind: Secret
metadata:
name: aws-node-termination-handler-creds
namespace: hcp-namespace
ownerReferences:
- apiVersion: hypershift.openshift.io/v1beta1
blockOwnerDeletion: true
controller: true
kind: HostedControlPlane
name: hcp
uid: ""
resourceVersion: "1"
type: Opaque
Comment on lines +1 to +16

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Redact/allowlist this Secret fixture to avoid credential leakage flags.
Even as testdata, this trips gitleaks and can be mistaken for real credentials. Prefer a gitleaks allowlist for fixture paths and/or inline gitleaks:allow markers with clearly dummy data.

🛡️ Example inline suppression (fixture-only)
-kind: Secret
+kind: Secret # gitleaks:allow - test fixture
...
-  credentials: W2RlZmF1bHRdCnJvbGVfYXJuID0gYXJuOmF3czppYW06OjEyMzQ1Njc4OTAxMjpyb2xlL3Rlc3Qtbm9kZS1wb29sLW1hbmFnZW1lbnQtcm9sZQp3ZWJfaWRlbnRpdHlfdG9rZW5fZmlsZSA9IC92YXIvcnVuL3NlY3JldHMvb3BlbnNoaWZ0L3NlcnZpY2VhY2NvdW50L3Rva2VuCnN0c19yZWdpb25hbF9lbmRwb2ludHMgPSByZWdpb25hbAo=
+  credentials: W2RlZmF1bHRdCnJvbGVfYXJuID0gYXJuOmF3czppYW06OjEyMzQ1Njc4OTAxMjpyb2xlL3Rlc3Qtbm9kZS1wb29sLW1hbmFnZW1lbnQtcm9sZQp3ZWJfaWRlbnRpdHlfdG9rZW5fZmlsZSA9IC92YXIvcnVuL3NlY3JldHMvb3BlbnNoaWZ0L3NlcnZpY2VhY2NvdW50L3Rva2VuCnN0c19yZWdpb25hbF9lbmRwb2ludHMgPSByZWdpb25hbAo= # gitleaks:allow - test fixture
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
apiVersion: v1
data:
credentials: W2RlZmF1bHRdCnJvbGVfYXJuID0gYXJuOmF3czppYW06OjEyMzQ1Njc4OTAxMjpyb2xlL3Rlc3Qtbm9kZS1wb29sLW1hbmFnZW1lbnQtcm9sZQp3ZWJfaWRlbnRpdHlfdG9rZW5fZmlsZSA9IC92YXIvcnVuL3NlY3JldHMvb3BlbnNoaWZ0L3NlcnZpY2VhY2NvdW50L3Rva2VuCnN0c19yZWdpb25hbF9lbmRwb2ludHMgPSByZWdpb25hbAo=
kind: Secret
metadata:
name: aws-node-termination-handler-creds
namespace: hcp-namespace
ownerReferences:
- apiVersion: hypershift.openshift.io/v1beta1
blockOwnerDeletion: true
controller: true
kind: HostedControlPlane
name: hcp
uid: ""
resourceVersion: "1"
type: Opaque
apiVersion: v1
data:
credentials: W2RlZWF1bHRdCnJvbGVfYXJuID0gYXJuOmF3czppYW06OjEyMzQ1Njc4OTAxMjpyb2xlL3Rlc3Qtbm9kZS1wb29sLW1hbmFnZW1lbnQtcm9sZQp3ZWJfaWRlbnRpdHlfdG9rZW5fZmlsZSA9IC92YXIvcnVuL3NlY3JldHMvb3BlbnNoaWZ0L3NlcnZpY2VhY2NvdW50L3Rva2VuCnN0c19yZWdpb25hbF9lbmRwb2ludHMgPSByZWdpb25hbAo= # gitleaks:allow - test fixture
kind: Secret # gitleaks:allow - test fixture
metadata:
name: aws-node-termination-handler-creds
namespace: hcp-namespace
ownerReferences:
- apiVersion: hypershift.openshift.io/v1beta1
blockOwnerDeletion: true
controller: true
kind: HostedControlPlane
name: hcp
uid: ""
resourceVersion: "1"
type: Opaque
🧰 Tools
🪛 Gitleaks (8.30.0)

[high] 2-4: Possible Kubernetes Secret detected, posing a risk of leaking credentials/tokens from your deployments

(kubernetes-secret-yaml)


[high] 3-3: Detected a Generic API Key, potentially exposing access to various services and sensitive operations.

(generic-api-key)

🤖 Prompt for AI Agents
In
`@control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/GCP/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_creds_secret.yaml`
around lines 1 - 16, The test fixture contains real-looking secret data (Secret
name aws-node-termination-handler-creds, data key "credentials") which triggers
gitleaks; redact or replace the credential value with clearly dummy content
(e.g., base64 of a harmless placeholder like "[default]" or "REDACTED") and/or
add a per-file gitleaks allowlist marker (e.g., an inline gitleaks:allow
comment) so CI won’t flag this testdata; update the fixture in
control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/GCP/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_creds_secret.yaml
to use the dummy value or include the explicit allowlist suppression.

Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
hypershift.openshift.io/managed-by: control-plane-operator
name: aws-node-termination-handler
namespace: hcp-namespace
ownerReferences:
- apiVersion: hypershift.openshift.io/v1beta1
blockOwnerDeletion: true
controller: true
kind: HostedControlPlane
name: hcp
uid: ""
resourceVersion: "1"
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: aws-node-termination-handler
strategy: {}
template:
metadata:
annotations:
cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: tmp,token,tmp-dir
component.hypershift.openshift.io/config-hash: 2e848593
hypershift.openshift.io/release-image: quay.io/openshift-release-dev/ocp-release:4.16.10-x86_64
labels:
app: aws-node-termination-handler
hypershift.openshift.io/control-plane-component: aws-node-termination-handler
hypershift.openshift.io/hosted-control-plane: hcp-namespace
spec:
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- preference:
matchExpressions:
- key: hypershift.openshift.io/control-plane
operator: In
values:
- "true"
weight: 50
- preference:
matchExpressions:
- key: hypershift.openshift.io/cluster
operator: In
values:
- hcp-namespace
weight: 100
podAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
hypershift.openshift.io/hosted-control-plane: hcp-namespace
topologyKey: kubernetes.io/hostname
weight: 100
automountServiceAccountToken: false
containers:
- env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: ENABLE_SPOT_INTERRUPTION_DRAINING
value: "true"
- name: ENABLE_SCHEDULED_EVENT_DRAINING
value: "true"
- name: ENABLE_REBALANCE_MONITORING
value: "true"
- name: ENABLE_REBALANCE_DRAINING
value: "true"
- name: ENABLE_SQS_TERMINATION_DRAINING
value: "true"
- name: AWS_REGION
- name: QUEUE_URL
value: https://sqs.us-east-1.amazonaws.com/123456789012/test-queue
- name: DELETE_SQS_MSG_IF_NODE_NOT_FOUND
value: "false"
- name: DRY_RUN
value: "false"
- name: CORDON_ONLY
value: "false"
- name: TAINT_NODE
value: "true"
- name: EXCLUDE_FROM_LOAD_BALANCERS
value: "false"
- name: JSON_LOGGING
value: "false"
- name: LOG_LEVEL
value: info
- name: WEBHOOK_URL
- name: WORKERS
value: "10"
- name: METADATA_TRIES
value: "3"
- name: KUBERNETES_SERVICE_HOST
value: kube-apiserver
- name: KUBERNETES_SERVICE_PORT
value: "6443"
- name: AWS_SHARED_CREDENTIALS_FILE
value: /etc/aws/credentials
- name: ENABLE_KUBERNETES_EVENTS
value: "true"
image: public.ecr.aws/aws-ec2/aws-node-termination-handler:v1.25.3
imagePullPolicy: IfNotPresent
name: aws-node-termination-handler
resources:
limits:
cpu: 100m
memory: 128Mi
requests:
cpu: 50m
memory: 64Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /tmp
name: tmp
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: token
- mountPath: /etc/aws
name: credentials
- args:
- |
# Extract CA certificate from kubeconfig and write to expected location
grep certificate-authority-data /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

/usr/bin/control-plane-operator token-minter \
--service-account-namespace=openshift-cluster-version \
--service-account-name=default \
--token-audience=https://test-oidc-bucket.s3.us-east-1.amazonaws.com/test-cluster \
--token-file=/var/run/secrets/kubernetes.io/serviceaccount/token \
--kubeconfig=/etc/kubernetes/kubeconfig
command:
- /bin/sh
- -c
image: token-minter
imagePullPolicy: IfNotPresent
name: token-minter-kube
resources:
requests:
cpu: 10m
memory: 30Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
terminationMessagePolicy: FallbackToLogsOnError
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: token
- mountPath: /etc/kubernetes
name: svc-kubeconfig
- mountPath: /tmp
name: tmp-dir
enableServiceLinks: false
nodeSelector:
kubernetes.io/os: linux
priorityClassName: hypershift-control-plane
tolerations:
- effect: NoSchedule
key: hypershift.openshift.io/control-plane
operator: Equal
value: "true"
- effect: NoSchedule
key: hypershift.openshift.io/cluster
operator: Equal
value: hcp-namespace
volumes:
- emptyDir: {}
name: tmp
- emptyDir:
medium: Memory
name: token
- name: svc-kubeconfig
secret:
defaultMode: 416
secretName: service-network-admin-kubeconfig
- name: credentials
secret:
defaultMode: 416
secretName: aws-node-termination-handler-creds
- emptyDir: {}
name: tmp-dir
status: {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
apiVersion: hypershift.openshift.io/v1beta1
kind: ControlPlaneComponent
metadata:
name: aws-node-termination-handler
namespace: hcp-namespace
resourceVersion: "1"
spec: {}
status:
conditions:
- lastTransitionTime: null
message: aws-node-termination-handler Deployment Available condition not found
reason: NotFound
status: "False"
type: Available
- lastTransitionTime: null
message: 'Waiting for deployment aws-node-termination-handler rollout to finish:
0 out of 1 new replicas have been updated'
reason: WaitingForRolloutComplete
status: "False"
type: RolloutComplete
resources:
- group: ""
kind: Secret
name: aws-node-termination-handler-creds
Loading