diff --git a/.go-version b/.go-version index eb716f77a7b8d..7a429d68a36af 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.24.9 +1.24.6 diff --git a/CHANGELOG/CHANGELOG-1.34.md b/CHANGELOG/CHANGELOG-1.34.md index 336f60a6e28cc..f731ad90ec210 100644 --- a/CHANGELOG/CHANGELOG-1.34.md +++ b/CHANGELOG/CHANGELOG-1.34.md @@ -1,256 +1,152 @@ -- [v1.34.1](#v1341) - - [Downloads for v1.34.1](#downloads-for-v1341) +- [v1.34.0](#v1340) + - [Downloads for v1.34.0](#downloads-for-v1340) - [Source Code](#source-code) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - [Container Images](#container-images) - - [Changelog since v1.34.0](#changelog-since-v1340) + - [Changelog since v1.33.0](#changelog-since-v1330) + - [Urgent Upgrade Notes](#urgent-upgrade-notes) + - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade) - [Changes by Kind](#changes-by-kind) + - [Deprecation](#deprecation) + - [API Change](#api-change) + - [Feature](#feature) + - [Failing Test](#failing-test) - [Bug or Regression](#bug-or-regression) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake) - [Dependencies](#dependencies) - [Added](#added) - [Changed](#changed) - [Removed](#removed) -- [v1.34.0](#v1340) - - [Downloads for v1.34.0](#downloads-for-v1340) +- [v1.34.0-rc.2](#v1340-rc2) + - [Downloads for v1.34.0-rc.2](#downloads-for-v1340-rc2) - [Source Code](#source-code-1) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - [Container Images](#container-images-1) - - [Changelog since v1.33.0](#changelog-since-v1330) - - [Urgent Upgrade Notes](#urgent-upgrade-notes) - - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade) + - [Changelog since v1.34.0-rc.1](#changelog-since-v1340-rc1) - [Changes by Kind](#changes-by-kind-1) - - [Deprecation](#deprecation) - - [API Change](#api-change) - - [Feature](#feature) - - [Failing Test](#failing-test) + - [Feature](#feature-1) + - [Documentation](#documentation) - [Bug or Regression](#bug-or-regression-1) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake) - [Dependencies](#dependencies-1) - [Added](#added-1) - [Changed](#changed-1) - [Removed](#removed-1) -- [v1.34.0-rc.2](#v1340-rc2) - - [Downloads for v1.34.0-rc.2](#downloads-for-v1340-rc2) +- [v1.34.0-rc.1](#v1340-rc1) + - [Downloads for v1.34.0-rc.1](#downloads-for-v1340-rc1) - [Source Code](#source-code-2) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) - [Container Images](#container-images-2) - - [Changelog since v1.34.0-rc.1](#changelog-since-v1340-rc1) + - [Changelog since v1.34.0-rc.0](#changelog-since-v1340-rc0) - [Changes by Kind](#changes-by-kind-2) - - [Feature](#feature-1) - - [Documentation](#documentation) - [Bug or Regression](#bug-or-regression-2) - [Dependencies](#dependencies-2) - [Added](#added-2) - [Changed](#changed-2) - [Removed](#removed-2) -- [v1.34.0-rc.1](#v1340-rc1) - - [Downloads for v1.34.0-rc.1](#downloads-for-v1340-rc1) +- [v1.34.0-rc.0](#v1340-rc0) + - [Downloads for v1.34.0-rc.0](#downloads-for-v1340-rc0) - [Source Code](#source-code-3) - [Client Binaries](#client-binaries-3) - [Server Binaries](#server-binaries-3) - [Node Binaries](#node-binaries-3) - [Container Images](#container-images-3) - - [Changelog since v1.34.0-rc.0](#changelog-since-v1340-rc0) + - [Changelog since v1.34.0-beta.0](#changelog-since-v1340-beta0) - [Changes by Kind](#changes-by-kind-3) + - [Deprecation](#deprecation-1) + - [API Change](#api-change-1) + - [Feature](#feature-2) + - [Failing Test](#failing-test-1) - [Bug or Regression](#bug-or-regression-3) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-1) - [Dependencies](#dependencies-3) - [Added](#added-3) - [Changed](#changed-3) - [Removed](#removed-3) -- [v1.34.0-rc.0](#v1340-rc0) - - [Downloads for v1.34.0-rc.0](#downloads-for-v1340-rc0) +- [v1.34.0-beta.0](#v1340-beta0) + - [Downloads for v1.34.0-beta.0](#downloads-for-v1340-beta0) - [Source Code](#source-code-4) - [Client Binaries](#client-binaries-4) - [Server Binaries](#server-binaries-4) - [Node Binaries](#node-binaries-4) - [Container Images](#container-images-4) - - [Changelog since v1.34.0-beta.0](#changelog-since-v1340-beta0) + - [Changelog since v1.34.0-alpha.3](#changelog-since-v1340-alpha3) - [Changes by Kind](#changes-by-kind-4) - - [Deprecation](#deprecation-1) - - [API Change](#api-change-1) - - [Feature](#feature-2) - - [Failing Test](#failing-test-1) + - [API Change](#api-change-2) + - [Feature](#feature-3) - [Bug or Regression](#bug-or-regression-4) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-1) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-2) - [Dependencies](#dependencies-4) - [Added](#added-4) - [Changed](#changed-4) - [Removed](#removed-4) -- [v1.34.0-beta.0](#v1340-beta0) - - [Downloads for v1.34.0-beta.0](#downloads-for-v1340-beta0) +- [v1.34.0-alpha.3](#v1340-alpha3) + - [Downloads for v1.34.0-alpha.3](#downloads-for-v1340-alpha3) - [Source Code](#source-code-5) - [Client Binaries](#client-binaries-5) - [Server Binaries](#server-binaries-5) - [Node Binaries](#node-binaries-5) - [Container Images](#container-images-5) - - [Changelog since v1.34.0-alpha.3](#changelog-since-v1340-alpha3) + - [Changelog since v1.34.0-alpha.2](#changelog-since-v1340-alpha2) - [Changes by Kind](#changes-by-kind-5) - - [API Change](#api-change-2) - - [Feature](#feature-3) + - [API Change](#api-change-3) + - [Feature](#feature-4) + - [Failing Test](#failing-test-2) - [Bug or Regression](#bug-or-regression-5) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-2) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-3) - [Dependencies](#dependencies-5) - [Added](#added-5) - [Changed](#changed-5) - [Removed](#removed-5) -- [v1.34.0-alpha.3](#v1340-alpha3) - - [Downloads for v1.34.0-alpha.3](#downloads-for-v1340-alpha3) +- [v1.34.0-alpha.2](#v1340-alpha2) + - [Downloads for v1.34.0-alpha.2](#downloads-for-v1340-alpha2) - [Source Code](#source-code-6) - [Client Binaries](#client-binaries-6) - [Server Binaries](#server-binaries-6) - [Node Binaries](#node-binaries-6) - [Container Images](#container-images-6) - - [Changelog since v1.34.0-alpha.2](#changelog-since-v1340-alpha2) + - [Changelog since v1.34.0-alpha.1](#changelog-since-v1340-alpha1) - [Changes by Kind](#changes-by-kind-6) - - [API Change](#api-change-3) - - [Feature](#feature-4) - - [Failing Test](#failing-test-2) + - [Deprecation](#deprecation-2) + - [API Change](#api-change-4) + - [Feature](#feature-5) - [Bug or Regression](#bug-or-regression-6) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-3) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-4) - [Dependencies](#dependencies-6) - [Added](#added-6) - [Changed](#changed-6) - [Removed](#removed-6) -- [v1.34.0-alpha.2](#v1340-alpha2) - - [Downloads for v1.34.0-alpha.2](#downloads-for-v1340-alpha2) +- [v1.34.0-alpha.1](#v1340-alpha1) + - [Downloads for v1.34.0-alpha.1](#downloads-for-v1340-alpha1) - [Source Code](#source-code-7) - [Client Binaries](#client-binaries-7) - [Server Binaries](#server-binaries-7) - [Node Binaries](#node-binaries-7) - [Container Images](#container-images-7) - - [Changelog since v1.34.0-alpha.1](#changelog-since-v1340-alpha1) - - [Changes by Kind](#changes-by-kind-7) - - [Deprecation](#deprecation-2) - - [API Change](#api-change-4) - - [Feature](#feature-5) - - [Bug or Regression](#bug-or-regression-7) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-4) - - [Dependencies](#dependencies-7) - - [Added](#added-7) - - [Changed](#changed-7) - - [Removed](#removed-7) -- [v1.34.0-alpha.1](#v1340-alpha1) - - [Downloads for v1.34.0-alpha.1](#downloads-for-v1340-alpha1) - - [Source Code](#source-code-8) - - [Client Binaries](#client-binaries-8) - - [Server Binaries](#server-binaries-8) - - [Node Binaries](#node-binaries-8) - - [Container Images](#container-images-8) - [Changelog since v1.33.0](#changelog-since-v1330-1) - [Urgent Upgrade Notes](#urgent-upgrade-notes-1) - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-1) - - [Changes by Kind](#changes-by-kind-8) + - [Changes by Kind](#changes-by-kind-7) - [Deprecation](#deprecation-3) - [API Change](#api-change-5) - [Feature](#feature-6) - [Failing Test](#failing-test-3) - - [Bug or Regression](#bug-or-regression-8) + - [Bug or Regression](#bug-or-regression-7) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-5) - - [Dependencies](#dependencies-8) - - [Added](#added-8) - - [Changed](#changed-8) - - [Removed](#removed-8) + - [Dependencies](#dependencies-7) + - [Added](#added-7) + - [Changed](#changed-7) + - [Removed](#removed-7) -# v1.34.1 - - -## Downloads for v1.34.1 - - - -### Source Code - -filename | sha512 hash --------- | ----------- -[kubernetes.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes.tar.gz) | b1262f114376f7bc0532ef688e758657ada0796e958c7b49e1401e8a2789791a7d59e5460c54780131fc8fa7398c6e87a7e59fdc4a84061c15d015c69a07e10d -[kubernetes-src.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-src.tar.gz) | 5109cd698bd249341357f5a0b7ab3cd078a641747ef1a17e168f650c62af854cc46bf3bca884f43ea33d51e81a2be4e31d0d02af639a3f58d79f3f1322b0e238 - -### Client Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-darwin-amd64.tar.gz) | c977b7ede3a07ec721a874ec127a9b2d2e1edce097e33fc5bfe0a7a2ecf61153c4e514787e89003eeb8d463f47ba0c09f3267669769f0cba873c5265674e056d -[kubernetes-client-darwin-arm64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-darwin-arm64.tar.gz) | ae6b112e45e50a9d1ce0738f948f933eed419dde20a70f399cfcf77ebf5179b6af893ae7e1e633f5b99c1f34a499a2238474cc45878afdf250c048ea43c559a2 -[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-linux-386.tar.gz) | 3e8aff795fa394343b4d3a943dba25b06b5c122df91fe5893cb354ee605a087f6150cee6225ff60d4b1ed9e0fa02adb9e4ccd8e38cd12337a92cedbdcfaabff2 -[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-linux-amd64.tar.gz) | 3abedd362fffd5eb749febdeb59c2edd9902f7f69fb182f879daeb27cc88405983c539513cb74ef9b9587ab3829bde992f22f2067fd181311989345f6e13b867 -[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-linux-arm.tar.gz) | 0d28e96ff4bf3f570277f194a975c19e8a1b49e7240908a91278647c44b5f019251dd7774aed5dbbfe7c030ded993701044c90ac97e14de5c51d0e9ae84d2127 -[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-linux-arm64.tar.gz) | 279832e1ac95532807aeb68ed951e8099300e3cd4a09f1d829c4b0197e0010d18d1de19e54f73b0ab7f104ee5670ef4897127432fac42867b7a727d75dc8bd48 -[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-linux-ppc64le.tar.gz) | 1367d4dfebab6f504612d6aa7e6dd7f6391ec28779c0610ef89c77bb691a5020ff3d863d5414645d62e9dfbf1fe814cf8b3bae3097c210f8e8ad895deb19c291 -[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-linux-s390x.tar.gz) | d03ff4bbad2c947a37a6ffc62f3db08cf2cc1d9d702d90b94f80fb9fdcc637c4f96096beb3a466f07ac4ca807d89e81240f15cf7d2ae1c6fbd4a953122728e28 -[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-windows-386.tar.gz) | 7929fd442acfa851c1510b52a6c3a11f6d3c2fb318597e68134a1927bac18ab70c6de7d572c0c05ecbc8c5764cf20fc91ab4c1ad604c7cd3707b86c01cb9fd16 -[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-windows-amd64.tar.gz) | f73e914d28e0986d4b32bbf0d39c428d3e4d28dac11cf8d2b48eae4f1825511fc8b1b706427a1fe752fc0d280f1b4c539f4261cc31f679f25646ac5234afa7ad -[kubernetes-client-windows-arm64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-client-windows-arm64.tar.gz) | f03de193bc851a1327cbc7338f019cabe7167775ca597c36637b10332b8892a7a4bcc5daa090349f24347f5210fced19c7a15211c69abb94fee87e88c1efaa30 - -### Server Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-server-linux-amd64.tar.gz) | 8fd1e779f4d0188592644e234a6e5b728b9000a2afeb9d8da25131a5a4e54718bb46c4d521c62e26ea971e32745529fbb001c4f011ef2c54091cb5e81b4b90f2 -[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-server-linux-arm64.tar.gz) | 77f68803b34f710c9623f388452494075ca9bb38567e7878176ec12a6d2971d2feba381e99462dc8c6e83ff5064dcffcaa7df736b67208880f5e90d71a831c2c -[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-server-linux-ppc64le.tar.gz) | 6a5378a02b9b27cce9e0bc26399f8c0a8676372407bb618949fa41caacb4bbfbc7ec5487e00d973fbf409abe848a3aed42b2ead2c78753a1dd7c3251daf61745 -[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-server-linux-s390x.tar.gz) | 6b9b4b64907ec817ce93a70faecbfcccf665e6b7681d0c21e26844c9d2645227ee8956c3b6b6a2417725b1e64353d5e1ed7071cf2c8e71ea8551cd47d662c3d8 - -### Node Binaries - -filename | sha512 hash --------- | ----------- -[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-node-linux-amd64.tar.gz) | c9b7d52708c4282757cd7aaa8b059c26f8f427cf8c238dff95cdc85a68d42c28b6e09fbf1aee3fa6f5f377aa395c6b9a73c112c56a6485e22b16a9c8562a8eef -[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-node-linux-arm64.tar.gz) | efe54933eb5e7e6b44c76efe0b4cec911340793ef2eafdd595593fb2537e5704429d3a291793cb69ad459fe14058da491a29a12d963ba34ee4c1475cc0799b0f -[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-node-linux-ppc64le.tar.gz) | 59a7223e167c890d8cb8544b9692182aaccb3814cb203337ea21a87902e0174d6f0e114d015989c42890d3b73cb73bdf8b1b71ef89fd1b0cf615349d10c23f8f -[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-node-linux-s390x.tar.gz) | b648658aaae4812d787b7be04bdfd13dc379316bbcda107eca410ffbdf57713f00bbb68ad4fe9501c3bb26e5d35f589653d4067a5753f681e41f493a28309ea9 -[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.34.1/kubernetes-node-windows-amd64.tar.gz) | 4c70f856364a976aa919662f3b3f6f06da3fe7ae156b7bf3fd84de4b5a0b0c70221283220c48c3cc31dddce0f2e0167606126515b1750ca90aaf129f1c9280ce - -### Container Images - -All container images are available as manifest lists and support the described -architectures. It is also possible to pull a specific architecture directly by -adding the "-$ARCH" suffix to the container image name. - -name | architectures ----- | ------------- -[registry.k8s.io/conformance:v1.34.1](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-s390x) -[registry.k8s.io/kube-apiserver:v1.34.1](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-s390x) -[registry.k8s.io/kube-controller-manager:v1.34.1](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-s390x) -[registry.k8s.io/kube-proxy:v1.34.1](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-s390x) -[registry.k8s.io/kube-scheduler:v1.34.1](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-s390x) -[registry.k8s.io/kubectl:v1.34.1](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-s390x) - -## Changelog since v1.34.0 - -## Changes by Kind - -### Bug or Regression - -- Fixed SELinux warning controller not emitting events on some SELinux label conflicts. ([#133745](https://github.com/kubernetes/kubernetes/pull/133745), [@jsafrane](https://github.com/jsafrane)) [SIG Apps, Storage and Testing] -- Fixed broken shell completion for api resources. ([#133783](https://github.com/kubernetes/kubernetes/pull/133783), [@vpnachev](https://github.com/vpnachev)) [SIG CLI] -- Kube-apiserver: Fixed a 1.34 regression in CustomResourceDefinition handling that incorrectly warned about unrecognized formats on number and integer properties ([#133901](https://github.com/kubernetes/kubernetes/pull/133901), [@yongruilin](https://github.com/yongruilin)) [SIG API Machinery] -- Kube-apiserver: Fixes a 1.34 regression with spurious "Error getting keys" log messages ([#133866](https://github.com/kubernetes/kubernetes/pull/133866), [@serathius](https://github.com/serathius)) [SIG API Machinery and Etcd] -- Kube-apiserver: Fixes a possible 1.34 performance regression calculating object size statistics for resources not served from the watch cache, typically only Events ([#133879](https://github.com/kubernetes/kubernetes/pull/133879), [@serathius](https://github.com/serathius)) [SIG API Machinery and Etcd] -- Kubeadm: fixed bug where v1beta3's ClusterConfiguration.APIServer.TimeoutForControlPlane is not respected in newer versions of kubeadm where v1beta4 is the default. ([#133753](https://github.com/kubernetes/kubernetes/pull/133753), [@HirazawaUi](https://github.com/HirazawaUi)) [SIG Cluster Lifecycle] - -## Dependencies - -### Added -_Nothing has changed._ - -### Changed -_Nothing has changed._ - -### Removed -_Nothing has changed._ - - - # v1.34.0 [Documentation](https://docs.k8s.io) diff --git a/build/build-image/cross/VERSION b/build/build-image/cross/VERSION index 14a4b84c2030d..fc8bcd43d9d76 100644 --- a/build/build-image/cross/VERSION +++ b/build/build-image/cross/VERSION @@ -1 +1 @@ -v1.34.0-go1.24.9-bullseye.0 +v1.34.0-go1.24.6-bullseye.0 diff --git a/build/common.sh b/build/common.sh index 2bd3cd052272c..38c20f5ffbb84 100755 --- a/build/common.sh +++ b/build/common.sh @@ -97,9 +97,9 @@ readonly KUBE_RSYNC_PORT="${KUBE_RSYNC_PORT:-}" readonly KUBE_CONTAINER_RSYNC_PORT=8730 # These are the default versions (image tags) for their respective base images. -readonly __default_distroless_iptables_version=v0.7.11 -readonly __default_go_runner_version=v2.4.0-go1.24.9-bookworm.0 -readonly __default_setcap_version=bookworm-v1.0.6 +readonly __default_distroless_iptables_version=v0.7.8 +readonly __default_go_runner_version=v2.4.0-go1.24.6-bookworm.0 +readonly __default_setcap_version=bookworm-v1.0.4 # These are the base images for the Docker-wrapped binaries. readonly KUBE_GORUNNER_IMAGE="${KUBE_GORUNNER_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/go-runner:$__default_go_runner_version}" diff --git a/build/dependencies.yaml b/build/dependencies.yaml index 3abc723bea0f9..d4c736f25ec27 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -64,7 +64,7 @@ dependencies: # etcd - name: "etcd" - version: 3.6.5 + version: 3.6.4 refPaths: - path: cluster/gce/manifests/etcd.manifest match: etcd_docker_tag|etcd_version @@ -74,6 +74,10 @@ dependencies: match: DefaultEtcdVersion = - path: hack/lib/etcd.sh match: ETCD_VERSION= + - path: staging/src/k8s.io/sample-apiserver/artifacts/example/deployment.yaml + match: gcr.io/etcd-development/etcd + - path: test/utils/image/manifest.go + match: configs\[Etcd\] = Config{list\.GcEtcdRegistry, "etcd", "\d+\.\d+.\d+(-(alpha|beta|rc).\d+)?(-\d+)?"} - name: "etcd-image" version: 3.6.4 @@ -113,7 +117,7 @@ dependencies: # Golang # TODO: this should really be eliminated and controlled by .go-version - name: "golang: upstream version" - version: 1.24.9 + version: 1.24.6 refPaths: - path: .go-version - path: build/build-image/cross/VERSION @@ -134,13 +138,13 @@ dependencies: match: minimum_go_version=go([0-9]+\.[0-9]+) - name: "registry.k8s.io/kube-cross: dependents" - version: v1.34.0-go1.24.9-bullseye.0 + version: v1.34.0-go1.24.6-bullseye.0 refPaths: - path: build/build-image/cross/VERSION # Base images - name: "registry.k8s.io/debian-base: dependents" - version: bookworm-v1.0.6 + version: bookworm-v1.0.4 refPaths: - path: cluster/images/etcd/Makefile match: BASEIMAGE\?\=registry\.k8s\.io\/build-image\/debian-base:[a-zA-Z]+\-v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?) @@ -170,7 +174,7 @@ dependencies: match: registry\.k8s\.io\/build-image\/debian-base:[a-zA-Z]+\-v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?) - name: "registry.k8s.io/distroless-iptables: dependents" - version: v0.7.11 + version: v0.7.8 refPaths: - path: build/common.sh match: __default_distroless_iptables_version= @@ -178,7 +182,7 @@ dependencies: match: configs\[DistrolessIptables\] = Config{list\.BuildImageRegistry, "distroless-iptables", "v([0-9]+)\.([0-9]+)\.([0-9]+)"} - name: "registry.k8s.io/go-runner: dependents" - version: v2.4.0-go1.24.9-bookworm.0 + version: v2.4.0-go1.24.6-bookworm.0 refPaths: - path: build/common.sh match: __default_go_runner_version= @@ -236,7 +240,7 @@ dependencies: match: configs\[Pause\] = Config{list\.GcRegistry, "pause", "\d+\.\d+(.\d+)?"} - name: "registry.k8s.io/build-image/setcap: dependents" - version: bookworm-v1.0.6 + version: bookworm-v1.0.4 refPaths: - path: build/common.sh match: __default_setcap_version= diff --git a/cluster/addons/addon-manager/Makefile b/cluster/addons/addon-manager/Makefile index 8727a80417d00..239fdaec6daab 100644 --- a/cluster/addons/addon-manager/Makefile +++ b/cluster/addons/addon-manager/Makefile @@ -18,7 +18,7 @@ TEMP_DIR:=$(shell mktemp -d) VERSION=v9.1.8 KUBECTL_VERSION?=v1.32.2 -BASEIMAGE=registry.k8s.io/build-image/debian-base-$(ARCH):bookworm-v1.0.6 +BASEIMAGE=registry.k8s.io/build-image/debian-base-$(ARCH):bookworm-v1.0.4 SUDO=$(if $(filter 0,$(shell id -u)),,sudo) diff --git a/cluster/common.sh b/cluster/common.sh index 1111f5ef418ee..448b3fe1218bb 100755 --- a/cluster/common.sh +++ b/cluster/common.sh @@ -481,13 +481,13 @@ EOF ;; server) echo "Generate server certificates..." - echo '{"CN":"'"${member_ip}"'","hosts":[],"key":{"algo":"ecdsa","size":256}}' \ + echo '{"CN":"'"${member_ip}"'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \ | ${CFSSL_BIN} gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server -hostname="${member_ip},127.0.0.1" - \ | ${CFSSLJSON_BIN} -bare "${prefix}" ;; peer) echo "Generate peer certificates..." - echo '{"CN":"'"${member_ip}"'","hosts":[],"key":{"algo":"ecdsa","size":256}}' \ + echo '{"CN":"'"${member_ip}"'","hosts":[""],"key":{"algo":"ecdsa","size":256}}' \ | ${CFSSL_BIN} gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer -hostname="${member_ip},127.0.0.1" - \ | ${CFSSLJSON_BIN} -bare "${prefix}" ;; diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index 2e26d07786c0c..ab7804bcfae1a 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -88,7 +88,7 @@ fi # By default, the latest image from the image family will be used unless an # explicit image will be set. GCI_VERSION=${KUBE_GCI_VERSION:-} -IMAGE_FAMILY=${KUBE_IMAGE_FAMILY:-cos-121-lts} +IMAGE_FAMILY=${KUBE_IMAGE_FAMILY:-cos-109-lts} export MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-} export MASTER_IMAGE_FAMILY=${KUBE_GCE_MASTER_IMAGE_FAMILY:-${IMAGE_FAMILY}} export MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud} diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh index 983898458478b..d950f67bddb6d 100755 --- a/cluster/gce/config-test.sh +++ b/cluster/gce/config-test.sh @@ -101,7 +101,7 @@ ALLOWED_NOTREADY_NODES=${ALLOWED_NOTREADY_NODES:-$(($(get-num-nodes) / 100))} # By default, the latest image from the image family will be used unless an # explicit image will be set. GCI_VERSION=${KUBE_GCI_VERSION:-} -IMAGE_FAMILY=${KUBE_IMAGE_FAMILY:-cos-121-lts} +IMAGE_FAMILY=${KUBE_IMAGE_FAMILY:-cos-109-lts} export MASTER_IMAGE=${KUBE_GCE_MASTER_IMAGE:-} export MASTER_IMAGE_FAMILY=${KUBE_GCE_MASTER_IMAGE_FAMILY:-${IMAGE_FAMILY}} export MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-cos-cloud} diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh index c57e8d253b785..c21361817f7ef 100755 --- a/cluster/gce/gci/configure-helper.sh +++ b/cluster/gce/gci/configure-helper.sh @@ -1963,14 +1963,10 @@ def resolve(host): fi sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}" sed -i -e "s@{{ *etcd_apiserver_protocol *}}@$etcd_apiserver_protocol@g" "${temp_file}" - - etcd_creds_and_extra_args="${etcd_creds} ${etcd_apiserver_creds} ${etcd_extra_args}" - etcd_creds_and_extra_args=$(echo "$etcd_creds_and_extra_args" | awk '{for (i=1;i<=NF;i++) printf "\"%s\"%s", $i, (i>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls {{ etcd_apiserver_protocol }}://127.0.0.1:{{ port }} --listen-client-urls {{ etcd_apiserver_protocol }}://{{ listen_client_ip }}:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1" + ], "env": [ { "name": "TARGET_STORAGE", "value": "{{ pillar.get('storage_backend', 'etcd3') }}" }, { "name": "TARGET_VERSION", - "value": "{{ pillar.get('etcd_version', '3.6.5') }}" + "value": "{{ pillar.get('etcd_version', '3.6.4') }}" }, { "name": "DO_NOT_MOVE_BINARIES", @@ -61,6 +52,12 @@ { "name": "INITIAL_ADVERTISE_PEER_URLS", "value": "{{ etcd_protocol }}://{{ hostname }}:{{ server_port }}" }, + { "name": "ETCD_CREDS", + "value": "{{ etcd_creds }}" + }, + { "name": "ETCD_APISERVER_CREDS", + "value": "{{ etcd_apiserver_creds }}" + }, { "name": "ETCD_SNAPSHOT_COUNT", "value": "10000" }, @@ -75,12 +72,9 @@ "livenessProbe": { "exec": { "command": [ - "/usr/local/bin/etcdctl", - "--endpoints=127.0.0.1:{{ port }}", - "--command-timeout=15s", - {{ etcdctl_certs }} - "endpoint", - "health" + "/bin/sh", + "-c", + "set -x; exec /usr/local/bin/etcdctl --endpoints=127.0.0.1:{{ port }} {{ etcdctl_certs }} --command-timeout=15s endpoint health" ] }, "initialDelaySeconds": {{ liveness_probe_initial_delay }}, diff --git a/cluster/gce/upgrade-aliases.sh b/cluster/gce/upgrade-aliases.sh index fb88cd80cf652..417f8a509f5ff 100755 --- a/cluster/gce/upgrade-aliases.sh +++ b/cluster/gce/upgrade-aliases.sh @@ -170,8 +170,8 @@ export KUBE_GCE_ENABLE_IP_ALIASES=true export SECONDARY_RANGE_NAME="pods-default" export STORAGE_BACKEND="etcd3" export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf" -export ETCD_IMAGE=3.6.5-0 -export ETCD_VERSION=3.6.5 +export ETCD_IMAGE=3.6.4-0 +export ETCD_VERSION=3.6.4 # Upgrade master with updated kube envs "${KUBE_ROOT}/cluster/gce/upgrade.sh" -M -l diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 98f2002aac141..72ad580306e50 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -1835,7 +1835,7 @@ function generate-certs { # make the config for the signer echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" # create the kubelet client cert with the correct groups - echo '{"CN":"kubelet","names":[{"O":"system:nodes"}],"hosts":[],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare kubelet + echo '{"CN":"kubelet","names":[{"O":"system:nodes"}],"hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare kubelet mv "kubelet-key.pem" "pki/private/kubelet.key" mv "kubelet.pem" "pki/issued/kubelet.crt" rm -f "kubelet.csr" @@ -1900,7 +1900,7 @@ function generate-aggregator-certs { # make the config for the signer echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" # create the aggregator client cert with the correct groups - echo '{"CN":"aggregator","hosts":[],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare proxy-client + echo '{"CN":"aggregator","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare proxy-client mv "proxy-client-key.pem" "pki/private/proxy-client.key" mv "proxy-client.pem" "pki/issued/proxy-client.crt" rm -f "proxy-client.csr" @@ -1961,7 +1961,7 @@ function generate-konnectivity-server-certs { # make the config for the signer echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" # create the konnectivity server cert with the correct groups - echo '{"CN":"konnectivity-server","hosts":[],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server + echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server rm -f "konnectivity-server.csr" # Make the agent <-> konnectivity server side certificates. @@ -1977,7 +1977,7 @@ function generate-konnectivity-server-certs { # make the config for the signer echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json" # create the konnectivity server cert with the correct groups - echo '{"CN":"koonectivity-server","hosts":[],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent + echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent rm -f "konnectivity-agent.csr" echo "completed main certificate section") &>"${cert_create_debug_output}" || true @@ -2039,7 +2039,7 @@ function generate-cloud-pvl-admission-certs { # make the config for the signer echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" # create the cloud-pvl-admission cert with the correct groups - echo '{"CN":"cloud-pvl-admission","hosts":[],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare cloud-pvl-admission + echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare cloud-pvl-admission rm -f "cloud-pvl-admission.csr" # Make the cloud-pvl-admission server side certificates. @@ -2055,7 +2055,7 @@ function generate-cloud-pvl-admission-certs { # make the config for the signer echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json" # create the cloud-pvl-admission server cert with the correct groups - echo '{"CN":"cloud-pvl-admission","hosts":[],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent + echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent rm -f "konnectivity-agent.csr" echo "completed main certificate section") &>"${cert_create_debug_output}" || true diff --git a/cluster/images/etcd/Makefile b/cluster/images/etcd/Makefile index 7178ef7333ef0..650b4c13dccc0 100644 --- a/cluster/images/etcd/Makefile +++ b/cluster/images/etcd/Makefile @@ -92,19 +92,19 @@ DOCKERFILE.windows = Dockerfile.windows DOCKERFILE := ${DOCKERFILE.${OS}} ifeq ($(ARCH),amd64) - BASEIMAGE?=registry.k8s.io/build-image/debian-base:bookworm-v1.0.6 + BASEIMAGE?=registry.k8s.io/build-image/debian-base:bookworm-v1.0.4 endif ifeq ($(ARCH),arm) - BASEIMAGE?=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.6 + BASEIMAGE?=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.4 endif ifeq ($(ARCH),arm64) - BASEIMAGE?=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.6 + BASEIMAGE?=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.4 endif ifeq ($(ARCH),ppc64le) - BASEIMAGE?=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.6 + BASEIMAGE?=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.4 endif ifeq ($(ARCH),s390x) - BASEIMAGE?=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.6 + BASEIMAGE?=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.4 endif BASE.windows = mcr.microsoft.com/windows/nanoserver diff --git a/cmd/kubeadm/app/cmd/certs.go b/cmd/kubeadm/app/cmd/certs.go index 1e5115a558dcd..f4e7d92454fb9 100644 --- a/cmd/kubeadm/app/cmd/certs.go +++ b/cmd/kubeadm/app/cmd/certs.go @@ -49,7 +49,6 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/util/errors" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/output" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) var ( @@ -347,10 +346,7 @@ func getInternalCfg(cfgPath string, client kubernetes.Interface, cfg kubeadmapiv // In case the user is not providing a custom config, try to get current config from the cluster. // NB. this operation should not block, because we want to allow certificate renewal also in case of not-working clusters if cfgPath == "" && client != nil { - getNodeRegistration := true - getAPIEndpoint := staticpodutil.IsControlPlaneNode() - getComponentConfigs := true - internalcfg, err := configutil.FetchInitConfigurationFromCluster(client, printer, logPrefix, getNodeRegistration, getAPIEndpoint, getComponentConfigs) + internalcfg, err := configutil.FetchInitConfigurationFromCluster(client, printer, logPrefix, false, false) if err == nil { printer.Println() // add empty line to separate the FetchInitConfigurationFromCluster output from the command output // certificate renewal or expiration checking doesn't depend on a running cluster, which means the CertificatesDir diff --git a/cmd/kubeadm/app/cmd/init.go b/cmd/kubeadm/app/cmd/init.go index e25b9dea49fa8..ba21ebf35eb18 100644 --- a/cmd/kubeadm/app/cmd/init.go +++ b/cmd/kubeadm/app/cmd/init.go @@ -19,11 +19,9 @@ package cmd import ( "fmt" "io" - "net" "os" "path/filepath" "slices" - "strconv" "github.com/spf13/cobra" flag "github.com/spf13/pflag" @@ -571,26 +569,24 @@ func (d *initData) Client() (clientset.Interface, error) { return d.client, nil } -// WaitControlPlaneClient returns a basic client used for the purpose of waiting -// for control plane components to report 'ok' on their respective health check endpoints. -// It uses the admin.conf as the base, but modifies it to point at the local API server instead -// of the control plane endpoint. -func (d *initData) WaitControlPlaneClient() (clientset.Interface, error) { - config, err := clientcmd.LoadFromFile(d.KubeConfigPath()) - if err != nil { - return nil, err - } - for _, v := range config.Clusters { - v.Server = fmt.Sprintf("https://%s", - net.JoinHostPort( - d.Cfg().LocalAPIEndpoint.AdvertiseAddress, - strconv.Itoa(int(d.Cfg().LocalAPIEndpoint.BindPort)), - ), - ) - } - client, err := kubeconfigutil.ToClientSet(config) - if err != nil { - return nil, err +// ClientWithoutBootstrap returns a dry-run client or a regular client from admin.conf. +// Unlike Client(), it does not call EnsureAdminClusterRoleBinding() or sets d.client. +// This means the client only has anonymous permissions and does not persist in initData. +func (d *initData) ClientWithoutBootstrap() (clientset.Interface, error) { + var ( + client clientset.Interface + err error + ) + if d.dryRun { + client, err = getDryRunClient(d) + if err != nil { + return nil, err + } + } else { // Use a real client + client, err = kubeconfigutil.ClientSetFromFile(d.KubeConfigPath()) + if err != nil { + return nil, err + } } return client, nil } diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go index 12fce4a1cc5ca..8916bc709108b 100644 --- a/cmd/kubeadm/app/cmd/join.go +++ b/cmd/kubeadm/app/cmd/join.go @@ -19,10 +19,8 @@ package cmd import ( "fmt" "io" - "net" "os" "path/filepath" - "strconv" "strings" "text/template" @@ -627,31 +625,6 @@ func (j *joinData) Client() (clientset.Interface, error) { return client, nil } -// WaitControlPlaneClient returns a basic client used for the purpose of waiting -// for control plane components to report 'ok' on their respective health check endpoints. -// It uses the admin.conf as the base, but modifies it to point at the local API server instead -// of the control plane endpoint. -func (j *joinData) WaitControlPlaneClient() (clientset.Interface, error) { - pathAdmin := filepath.Join(j.KubeConfigDir(), kubeadmconstants.AdminKubeConfigFileName) - config, err := clientcmd.LoadFromFile(pathAdmin) - if err != nil { - return nil, err - } - for _, v := range config.Clusters { - v.Server = fmt.Sprintf("https://%s", - net.JoinHostPort( - j.Cfg().ControlPlane.LocalAPIEndpoint.AdvertiseAddress, - strconv.Itoa(int(j.Cfg().ControlPlane.LocalAPIEndpoint.BindPort)), - ), - ) - } - client, err := kubeconfigutil.ToClientSet(config) - if err != nil { - return nil, err - } - return client, nil -} - // IgnorePreflightErrors returns the list of preflight errors to ignore. func (j *joinData) IgnorePreflightErrors() sets.Set[string] { return j.ignorePreflightErrors @@ -692,10 +665,7 @@ func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfigurati } // Create the final KubeConfig file with the cluster name discovered after fetching the cluster configuration - _, clusterinfo, err := kubeconfigutil.GetClusterFromKubeConfig(tlsBootstrapCfg) - if err != nil { - return nil, errors.Wrap(err, "the TLS bootstrap kubeconfig is malformed") - } + _, clusterinfo := kubeconfigutil.GetClusterFromKubeConfig(tlsBootstrapCfg) tlsBootstrapCfg.Clusters = map[string]*clientcmdapi.Cluster{ initConfiguration.ClusterName: clusterinfo, } @@ -712,10 +682,7 @@ func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfigurati // fetchInitConfiguration reads the cluster configuration from the kubeadm-admin configMap func fetchInitConfiguration(client clientset.Interface) (*kubeadmapi.InitConfiguration, error) { - getNodeRegistration := false - getAPIEndpoint := false - getComponentConfigs := true - initConfiguration, err := configutil.FetchInitConfigurationFromCluster(client, nil, "preflight", getNodeRegistration, getAPIEndpoint, getComponentConfigs) + initConfiguration, err := configutil.FetchInitConfigurationFromCluster(client, nil, "preflight", true, false) if err != nil { return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") } diff --git a/cmd/kubeadm/app/cmd/phases/init/data.go b/cmd/kubeadm/app/cmd/phases/init/data.go index 6e7b8ba3a6b48..21bf0778039e0 100644 --- a/cmd/kubeadm/app/cmd/phases/init/data.go +++ b/cmd/kubeadm/app/cmd/phases/init/data.go @@ -47,7 +47,7 @@ type InitData interface { ExternalCA() bool OutputWriter() io.Writer Client() (clientset.Interface, error) - WaitControlPlaneClient() (clientset.Interface, error) + ClientWithoutBootstrap() (clientset.Interface, error) Tokens() []string PatchesDir() string } diff --git a/cmd/kubeadm/app/cmd/phases/init/data_test.go b/cmd/kubeadm/app/cmd/phases/init/data_test.go index 8465021446ada..8229cfc76c897 100644 --- a/cmd/kubeadm/app/cmd/phases/init/data_test.go +++ b/cmd/kubeadm/app/cmd/phases/init/data_test.go @@ -50,6 +50,6 @@ func (t *testInitData) KubeletDir() string { r func (t *testInitData) ExternalCA() bool { return false } func (t *testInitData) OutputWriter() io.Writer { return nil } func (t *testInitData) Client() (clientset.Interface, error) { return nil, nil } -func (t *testInitData) WaitControlPlaneClient() (clientset.Interface, error) { return nil, nil } +func (t *testInitData) ClientWithoutBootstrap() (clientset.Interface, error) { return nil, nil } func (t *testInitData) Tokens() []string { return nil } func (t *testInitData) PatchesDir() string { return "" } diff --git a/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go index c93d02aef1c56..51308db28961b 100644 --- a/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go +++ b/cmd/kubeadm/app/cmd/phases/init/waitcontrolplane.go @@ -58,7 +58,8 @@ func runWaitControlPlanePhase(c workflow.RunData) error { } } - client, err := data.WaitControlPlaneClient() + // Both Wait* calls below use a /healthz endpoint, thus a client without permissions works fine + client, err := data.ClientWithoutBootstrap() if err != nil { return errors.Wrap(err, "cannot obtain client without bootstrap") } diff --git a/cmd/kubeadm/app/cmd/phases/join/data.go b/cmd/kubeadm/app/cmd/phases/join/data.go index f6b3e769b22e8..8005a6b07fd5d 100644 --- a/cmd/kubeadm/app/cmd/phases/join/data.go +++ b/cmd/kubeadm/app/cmd/phases/join/data.go @@ -38,7 +38,6 @@ type JoinData interface { TLSBootstrapCfg() (*clientcmdapi.Config, error) InitCfg() (*kubeadmapi.InitConfiguration, error) Client() (clientset.Interface, error) - WaitControlPlaneClient() (clientset.Interface, error) IgnorePreflightErrors() sets.Set[string] OutputWriter() io.Writer PatchesDir() string diff --git a/cmd/kubeadm/app/cmd/phases/join/data_test.go b/cmd/kubeadm/app/cmd/phases/join/data_test.go index 5995acf7d7356..c03e06bea1b91 100644 --- a/cmd/kubeadm/app/cmd/phases/join/data_test.go +++ b/cmd/kubeadm/app/cmd/phases/join/data_test.go @@ -32,17 +32,16 @@ type testJoinData struct{} // testJoinData must satisfy JoinData. var _ JoinData = &testJoinData{} -func (j *testJoinData) CertificateKey() string { return "" } -func (j *testJoinData) Cfg() *kubeadmapi.JoinConfiguration { return nil } -func (j *testJoinData) TLSBootstrapCfg() (*clientcmdapi.Config, error) { return nil, nil } -func (j *testJoinData) InitCfg() (*kubeadmapi.InitConfiguration, error) { return nil, nil } -func (j *testJoinData) Client() (clientset.Interface, error) { return nil, nil } -func (j *testJoinData) WaitControlPlaneClient() (clientset.Interface, error) { return nil, nil } -func (j *testJoinData) IgnorePreflightErrors() sets.Set[string] { return nil } -func (j *testJoinData) OutputWriter() io.Writer { return nil } -func (j *testJoinData) PatchesDir() string { return "" } -func (j *testJoinData) DryRun() bool { return false } -func (j *testJoinData) KubeConfigDir() string { return "" } -func (j *testJoinData) KubeletDir() string { return "" } -func (j *testJoinData) ManifestDir() string { return "" } -func (j *testJoinData) CertificateWriteDir() string { return "" } +func (j *testJoinData) CertificateKey() string { return "" } +func (j *testJoinData) Cfg() *kubeadmapi.JoinConfiguration { return nil } +func (j *testJoinData) TLSBootstrapCfg() (*clientcmdapi.Config, error) { return nil, nil } +func (j *testJoinData) InitCfg() (*kubeadmapi.InitConfiguration, error) { return nil, nil } +func (j *testJoinData) Client() (clientset.Interface, error) { return nil, nil } +func (j *testJoinData) IgnorePreflightErrors() sets.Set[string] { return nil } +func (j *testJoinData) OutputWriter() io.Writer { return nil } +func (j *testJoinData) PatchesDir() string { return "" } +func (j *testJoinData) DryRun() bool { return false } +func (j *testJoinData) KubeConfigDir() string { return "" } +func (j *testJoinData) KubeletDir() string { return "" } +func (j *testJoinData) ManifestDir() string { return "" } +func (j *testJoinData) CertificateWriteDir() string { return "" } diff --git a/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go b/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go index bd29eabbf60cd..ba4bb82b346c3 100644 --- a/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go +++ b/cmd/kubeadm/app/cmd/phases/join/waitcontrolplane.go @@ -50,7 +50,7 @@ func runWaitControlPlanePhase(c workflow.RunData) error { return nil } - client, err := data.WaitControlPlaneClient() + client, err := data.Client() if err != nil { return err } diff --git a/cmd/kubeadm/app/cmd/reset.go b/cmd/kubeadm/app/cmd/reset.go index 23f2bf62a1c56..818b6eb276836 100644 --- a/cmd/kubeadm/app/cmd/reset.go +++ b/cmd/kubeadm/app/cmd/reset.go @@ -44,7 +44,6 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/util/errors" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) var ( @@ -133,10 +132,7 @@ func newResetData(cmd *cobra.Command, opts *resetOptions, in io.Reader, out io.W if err == nil { klog.V(1).Infof("[reset] Loaded client set from kubeconfig file: %s", opts.kubeconfigPath) - getNodeRegistration := true - getAPIEndpoint := staticpodutil.IsControlPlaneNode() - getComponentConfigs := true - initCfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", getNodeRegistration, getAPIEndpoint, getComponentConfigs) + initCfg, err = configutil.FetchInitConfigurationFromCluster(client, nil, "reset", false, false) if err != nil { klog.Warningf("[reset] Unable to fetch the kubeadm-config ConfigMap from cluster: %v", err) } diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go index c6c61ed3ac621..a7bae51dd0304 100644 --- a/cmd/kubeadm/app/cmd/upgrade/apply.go +++ b/cmd/kubeadm/app/cmd/upgrade/apply.go @@ -229,10 +229,7 @@ func newApplyData(cmd *cobra.Command, args []string, applyFlags *applyFlags) (*a // Fetches the cluster configuration. klog.V(1).Infoln("[upgrade] retrieving configuration from cluster") - getNodeRegistration := true - isControlPlaneNode := true - getComponentConfigs := true - initCfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", getNodeRegistration, isControlPlaneNode, getComponentConfigs) + initCfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", false, false) if err != nil { if apierrors.IsNotFound(err) { _, _ = printer.Printf("[upgrade] In order to upgrade, a ConfigMap called %q in the %q namespace must exist.\n", constants.KubeadmConfigConfigMap, metav1.NamespaceSystem) diff --git a/cmd/kubeadm/app/cmd/upgrade/common.go b/cmd/kubeadm/app/cmd/upgrade/common.go index 0fee21e2ecbd7..a3579e532540e 100644 --- a/cmd/kubeadm/app/cmd/upgrade/common.go +++ b/cmd/kubeadm/app/cmd/upgrade/common.go @@ -45,7 +45,6 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/util/errors" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/output" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) // enforceRequirements verifies that it's okay to upgrade and then returns the variables needed for the rest of the procedure @@ -93,10 +92,7 @@ func enforceRequirements(flagSet *pflag.FlagSet, flags *applyPlanFlags, args []s return nil, nil, nil, nil, err } - getNodeRegistration := true - getAPIEndpoint := staticpodutil.IsControlPlaneNode() - getComponentConfigs := true - initCfg, err := configutil.FetchInitConfigurationFromCluster(client, printer, "upgrade/config", getNodeRegistration, getAPIEndpoint, getComponentConfigs) + initCfg, err := configutil.FetchInitConfigurationFromCluster(client, printer, "upgrade/config", false, false) if err != nil { return nil, nil, nil, nil, errors.Wrap(err, "[upgrade/init config] FATAL") } diff --git a/cmd/kubeadm/app/cmd/upgrade/diff.go b/cmd/kubeadm/app/cmd/upgrade/diff.go index 3d5bf468e9b8c..947d763a57b43 100644 --- a/cmd/kubeadm/app/cmd/upgrade/diff.go +++ b/cmd/kubeadm/app/cmd/upgrade/diff.go @@ -41,7 +41,6 @@ import ( "k8s.io/kubernetes/cmd/kubeadm/app/util/errors" kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig" "k8s.io/kubernetes/cmd/kubeadm/app/util/output" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) type diffFlags struct { @@ -107,7 +106,7 @@ func validateManifestsPath(manifests ...string) (err error) { } // FetchInitConfigurationFunc defines the signature of the function which will fetch InitConfiguration from cluster. -type FetchInitConfigurationFunc func(client clientset.Interface, printer output.Printer, logPrefix string, getNodeRegistration, getAPIEndpoint, getComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) +type FetchInitConfigurationFunc func(client clientset.Interface, printer output.Printer, logPrefix string, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) func runDiff(fs *pflag.FlagSet, flags *diffFlags, args []string, fetchInitConfigurationFromCluster FetchInitConfigurationFunc) error { externalCfg := &v1beta4.UpgradeConfiguration{} @@ -120,10 +119,7 @@ func runDiff(fs *pflag.FlagSet, flags *diffFlags, args []string, fetchInitConfig if err != nil { return errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath) } - getNodeRegistration := true - getAPIEndpoint := staticpodutil.IsControlPlaneNode() - getComponentConfigs := false - initCfg, err := fetchInitConfigurationFromCluster(client, &output.TextPrinter{}, "upgrade/diff", getNodeRegistration, getAPIEndpoint, getComponentConfigs) + initCfg, err := fetchInitConfigurationFromCluster(client, &output.TextPrinter{}, "upgrade/diff", false, true) if err != nil { return err } diff --git a/cmd/kubeadm/app/cmd/upgrade/diff_test.go b/cmd/kubeadm/app/cmd/upgrade/diff_test.go index 7e65ee1a18bf2..f23041285926e 100644 --- a/cmd/kubeadm/app/cmd/upgrade/diff_test.go +++ b/cmd/kubeadm/app/cmd/upgrade/diff_test.go @@ -44,7 +44,7 @@ func createTestRunDiffFile(contents []byte) (string, error) { return file.Name(), nil } -func fakeFetchInitConfig(client clientset.Interface, printer output.Printer, logPrefix string, getNodeRegistration, getAPIEndpoint, getComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { +func fakeFetchInitConfig(client clientset.Interface, printer output.Printer, logPrefix string, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { return &kubeadmapi.InitConfiguration{ ClusterConfiguration: kubeadmapi.ClusterConfiguration{ KubernetesVersion: "v1.0.1", diff --git a/cmd/kubeadm/app/cmd/upgrade/node.go b/cmd/kubeadm/app/cmd/upgrade/node.go index 1e3cd71041e8a..b0f57d0b1e28e 100644 --- a/cmd/kubeadm/app/cmd/upgrade/node.go +++ b/cmd/kubeadm/app/cmd/upgrade/node.go @@ -19,12 +19,14 @@ package upgrade import ( "fmt" "io" + "os" "github.com/spf13/cobra" flag "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4" @@ -38,7 +40,6 @@ import ( configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config" "k8s.io/kubernetes/cmd/kubeadm/app/util/errors" "k8s.io/kubernetes/cmd/kubeadm/app/util/output" - staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod" ) // nodeOptions defines all the options exposed via flags by kubeadm upgrade node. @@ -163,8 +164,13 @@ func addUpgradeNodeFlags(flagSet *flag.FlagSet, nodeOptions *nodeOptions) { // This func takes care of validating nodeOptions passed to the command, and then it converts // options into the internal InitConfiguration type that is used as input all the phases in the kubeadm upgrade node workflow func newNodeData(cmd *cobra.Command, nodeOptions *nodeOptions, out io.Writer) (*nodeData, error) { - isControlPlaneNode := staticpodutil.IsControlPlaneNode() - + // Checks if a node is a control-plane node by looking up the kube-apiserver manifest file + isControlPlaneNode := true + filepath := constants.GetStaticPodFilepath(constants.KubeAPIServer, constants.GetStaticPodDirectory()) + if _, err := os.Stat(filepath); os.IsNotExist(err) { + klog.V(1).Infof("assuming this is not a control plane node because %q is missing", filepath) + isControlPlaneNode = false + } if len(nodeOptions.kubeConfigPath) == 0 { // Update the kubeconfig path depending on whether this is a control plane node or not. nodeOptions.kubeConfigPath = constants.GetKubeletKubeConfigPath() @@ -192,10 +198,9 @@ func newNodeData(cmd *cobra.Command, nodeOptions *nodeOptions, out io.Writer) (* } // Fetches the cluster configuration - getNodeRegistration := true - getAPIEndpoint := isControlPlaneNode - getComponentConfigs := true - initCfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", getNodeRegistration, getAPIEndpoint, getComponentConfigs) + // NB in case of control-plane node, we are reading all the info for the node; in case of NOT control-plane node + // (worker node), we are not reading local API address and the CRI socket from the node object + initCfg, err := configutil.FetchInitConfigurationFromCluster(client, nil, "upgrade", !isControlPlaneNode, false) if err != nil { return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap") } diff --git a/cmd/kubeadm/app/cmd/util/join.go b/cmd/kubeadm/app/cmd/util/join.go index aad2e4dff0f36..feded1d06d75c 100644 --- a/cmd/kubeadm/app/cmd/util/join.go +++ b/cmd/kubeadm/app/cmd/util/join.go @@ -56,9 +56,9 @@ func getJoinCommand(kubeConfigFile, token, key string, controlPlane, skipTokenPr } // load the default cluster config - _, clusterConfig, err := kubeconfigutil.GetClusterFromKubeConfig(config) - if err != nil { - return "", errors.Wrapf(err, "malformed kubeconfig file: %s", kubeConfigFile) + _, clusterConfig := kubeconfigutil.GetClusterFromKubeConfig(config) + if clusterConfig == nil { + return "", errors.New("failed to get default cluster config") } // load CA certificates from the kubeconfig (either from PEM data or by file path) diff --git a/cmd/kubeadm/app/cmd/util/join_test.go b/cmd/kubeadm/app/cmd/util/join_test.go index d883ac0479da4..c9d08f3b17abc 100644 --- a/cmd/kubeadm/app/cmd/util/join_test.go +++ b/cmd/kubeadm/app/cmd/util/join_test.go @@ -133,7 +133,7 @@ func TestGetJoinCommand(t *testing.T) { kubeConfig: &clientcmdapi.Config{}, token: "test-token", expectError: true, - errorMessage: "the current context is invalid", + errorMessage: "failed to get default cluster config", }, { name: "Error when CA certificate is invalid", diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 4935f0530bfbc..21e49e24e6e4e 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -323,10 +323,10 @@ const ( KubeletHealthzPort = 10248 // MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports - MinExternalEtcdVersion = "3.5.24-0" + MinExternalEtcdVersion = "3.5.21-0" // DefaultEtcdVersion indicates the default etcd version that kubeadm uses - DefaultEtcdVersion = "3.6.5-0" + DefaultEtcdVersion = "3.6.4-0" // Etcd defines variable used internally when referring to etcd component Etcd = "etcd" @@ -498,10 +498,10 @@ var ( // SupportedEtcdVersion lists officially supported etcd versions with corresponding Kubernetes releases SupportedEtcdVersion = map[uint8]string{ - 31: "3.5.24-0", - 32: "3.5.24-0", - 33: "3.5.24-0", - 34: "3.6.5-0", + 31: "3.5.21-0", + 32: "3.5.21-0", + 33: "3.5.21-0", + 34: "3.6.4-0", } // KubeadmCertsClusterRoleName sets the name for the ClusterRole that allows diff --git a/cmd/kubeadm/app/discovery/discovery.go b/cmd/kubeadm/app/discovery/discovery.go index ae51a902dc0d8..dbb45a5389090 100644 --- a/cmd/kubeadm/app/discovery/discovery.go +++ b/cmd/kubeadm/app/discovery/discovery.go @@ -21,7 +21,6 @@ import ( clientset "k8s.io/client-go/kubernetes" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - bootstrapapi "k8s.io/cluster-bootstrap/token/api" "k8s.io/klog/v2" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" @@ -52,10 +51,7 @@ func For(client clientset.Interface, cfg *kubeadmapi.JoinConfiguration) (*client if len(cfg.Discovery.TLSBootstrapToken) != 0 { klog.V(1).Info("[discovery] Using provided TLSBootstrapToken as authentication credentials for the join process") - _, clusterinfo, err := kubeconfigutil.GetClusterFromKubeConfig(config) - if err != nil { - return nil, errors.Wrapf(err, "malformed kubeconfig in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) - } + _, clusterinfo := kubeconfigutil.GetClusterFromKubeConfig(config) return kubeconfigutil.CreateWithToken( clusterinfo.Server, kubeadmapiv1.DefaultClusterName, diff --git a/cmd/kubeadm/app/discovery/file/file.go b/cmd/kubeadm/app/discovery/file/file.go index 2cc7caeb269a2..4a5cf3801cbf6 100644 --- a/cmd/kubeadm/app/discovery/file/file.go +++ b/cmd/kubeadm/app/discovery/file/file.go @@ -52,9 +52,9 @@ func ValidateConfigInfo(config *clientcmdapi.Config, discoveryTimeout time.Durat if len(config.Clusters) < 1 { return nil, errors.New("the provided kubeconfig file must have at least one Cluster defined") } - currentClusterName, currentCluster, err := kubeconfigutil.GetClusterFromKubeConfig(config) - if err != nil { - return nil, errors.Wrap(err, "the provided kubeconfig file is malformed") + currentClusterName, currentCluster := kubeconfigutil.GetClusterFromKubeConfig(config) + if currentCluster == nil { + return nil, errors.New("the provided kubeconfig file must have a unnamed Cluster or a CurrentContext that specifies a non-nil Cluster") } if err := clientcmd.Validate(*config); err != nil { return nil, err @@ -124,10 +124,7 @@ func ValidateConfigInfo(config *clientcmdapi.Config, discoveryTimeout time.Durat return config, nil } - _, refreshedCluster, err := kubeconfigutil.GetClusterFromKubeConfig(refreshedBaseKubeConfig) - if err != nil { - return nil, errors.Wrapf(err, "malformed kubeconfig in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) - } + _, refreshedCluster := kubeconfigutil.GetClusterFromKubeConfig(refreshedBaseKubeConfig) if currentCluster.Server != refreshedCluster.Server { klog.Warningf("[discovery] the API Server endpoint %q in use is different from the endpoint %q which defined in the %s ConfigMap", currentCluster.Server, refreshedCluster.Server, bootstrapapi.ConfigMapClusterInfo) } diff --git a/cmd/kubeadm/app/discovery/token/token.go b/cmd/kubeadm/app/discovery/token/token.go index 195678b143006..c05be648896b8 100644 --- a/cmd/kubeadm/app/discovery/token/token.go +++ b/cmd/kubeadm/app/discovery/token/token.go @@ -103,9 +103,9 @@ func retrieveValidatedConfigInfo(client clientset.Interface, cfg *kubeadmapi.Dis return nil, errors.Wrapf(err, "couldn't parse the kubeconfig file in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) } - _, _, err = kubeconfigutil.GetClusterFromKubeConfig(insecureConfig) - if err != nil { - return nil, errors.Wrapf(err, "malformed kubeconfig in the %s ConfigMap", bootstrapapi.ConfigMapClusterInfo) + // The ConfigMap should contain a single cluster + if len(insecureConfig.Clusters) != 1 { + return nil, errors.Errorf("expected the kubeconfig file in the %s ConfigMap to have a single cluster, but it had %d", bootstrapapi.ConfigMapClusterInfo, len(insecureConfig.Clusters)) } // If no TLS root CA pinning was specified, we're done diff --git a/cmd/kubeadm/app/discovery/token/token_test.go b/cmd/kubeadm/app/discovery/token/token_test.go index 2614662dcead4..a823af66ccd8d 100644 --- a/cmd/kubeadm/app/discovery/token/token_test.go +++ b/cmd/kubeadm/app/discovery/token/token_test.go @@ -77,12 +77,10 @@ users: null name string tokenID string tokenSecret string - currentContextCluster string cfg *kubeadmapi.Discovery configMap *fakeConfigMap delayedJWSSignaturePatch bool expectedError bool - expectedErrorString string }{ { // This is the default behavior. The JWS signature is patched after the cluster-info ConfigMap is created @@ -132,24 +130,6 @@ users: null data: nil, }, }, - { - name: "invalid: the kubeconfig in the configmap has the wrong current context", - tokenID: "123456", - tokenSecret: "abcdef1234567890", - cfg: &kubeadmapi.Discovery{ - BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ - Token: "123456.abcdef1234567890", - CACertHashes: []string{caCertHash}, - }, - }, - configMap: &fakeConfigMap{ - name: bootstrapapi.ConfigMapClusterInfo, - data: nil, - }, - currentContextCluster: "foo", - expectedError: true, - expectedErrorString: `malformed kubeconfig in the cluster-info ConfigMap: no matching cluster for the current context: token-bootstrap-client@somecluster`, - }, { name: "invalid: token format is invalid", tokenID: "foo", @@ -236,13 +216,6 @@ users: null for _, test := range tests { t.Run(test.name, func(t *testing.T) { kubeconfig := buildSecureBootstrapKubeConfig("127.0.0.1", []byte(caCert), "somecluster") - if len(test.currentContextCluster) > 0 { - currentContext := kubeconfig.Contexts[kubeconfig.CurrentContext] - if currentContext == nil { - t.Fatal("unexpected nil current context") - } - currentContext.Cluster = test.currentContextCluster - } kubeconfigBytes, err := clientcmd.Write(*kubeconfig) if err != nil { t.Fatalf("cannot marshal kubeconfig %v", err) @@ -294,13 +267,8 @@ users: null t.Errorf("expected error %v, got %v, error: %v", test.expectedError, err != nil, err) } + // Return if an error is expected if err != nil { - if len(test.expectedErrorString) > 0 && test.expectedErrorString != err.Error() { - t.Fatalf("expected error string: %s, got: %s", - test.expectedErrorString, err.Error()) - } - - // Return if an error is expected return } diff --git a/cmd/kubeadm/app/preflight/checks.go b/cmd/kubeadm/app/preflight/checks.go index bc99a641bb66d..36978405d19a8 100644 --- a/cmd/kubeadm/app/preflight/checks.go +++ b/cmd/kubeadm/app/preflight/checks.go @@ -463,8 +463,11 @@ func (subnet HTTPProxyCIDRCheck) Check() (warnings, errorList []error) { return nil, []error{errors.Wrapf(err, "unable to get first IP address from the given CIDR (%s)", cidr.String())} } - testHostString := net.JoinHostPort(testIP.String(), "1234") - url := fmt.Sprintf("%s://%s/", subnet.Proto, testHostString) + testIPstring := testIP.String() + if len(testIP) == net.IPv6len { + testIPstring = fmt.Sprintf("[%s]:1234", testIP) + } + url := fmt.Sprintf("%s://%s/", subnet.Proto, testIPstring) req, err := http.NewRequest("GET", url, nil) if err != nil { diff --git a/cmd/kubeadm/app/util/apiclient/wait.go b/cmd/kubeadm/app/util/apiclient/wait.go index 3938f35ec34ca..af8fa7e9f4126 100644 --- a/cmd/kubeadm/app/util/apiclient/wait.go +++ b/cmd/kubeadm/app/util/apiclient/wait.go @@ -23,7 +23,6 @@ import ( "io" "net" "net/http" - "strconv" "strings" "text/template" "time" @@ -358,8 +357,7 @@ func (w *KubeWaiter) WaitForKubelet(healthzAddress string, healthzPort int32) er var ( lastError error start = time.Now() - addrPort = net.JoinHostPort(healthzAddress, strconv.Itoa(int(healthzPort))) - healthzEndpoint = fmt.Sprintf("http://%s/healthz", addrPort) + healthzEndpoint = fmt.Sprintf("http://%s:%d/healthz", healthzAddress, healthzPort) ) if healthzPort == 0 { diff --git a/cmd/kubeadm/app/util/config/cluster.go b/cmd/kubeadm/app/util/config/cluster.go index 40ba9b2dbd188..724530f692a75 100644 --- a/cmd/kubeadm/app/util/config/cluster.go +++ b/cmd/kubeadm/app/util/config/cluster.go @@ -52,7 +52,7 @@ import ( ) // FetchInitConfigurationFromCluster fetches configuration from a ConfigMap in the cluster -func FetchInitConfigurationFromCluster(client clientset.Interface, printer output.Printer, logPrefix string, getNodeRegistration, getAPIEndpoint, getComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { +func FetchInitConfigurationFromCluster(client clientset.Interface, printer output.Printer, logPrefix string, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { if printer == nil { printer = &output.TextPrinter{} } @@ -61,7 +61,7 @@ func FetchInitConfigurationFromCluster(client clientset.Interface, printer outpu _, _ = printer.Printf("[%s] Use 'kubeadm init phase upload-config kubeadm --config your-config-file' to re-upload it.\n", logPrefix) // Fetch the actual config from cluster - cfg, err := getInitConfigurationFromCluster(constants.KubernetesDir, client, getNodeRegistration, getAPIEndpoint, getComponentConfigs) + cfg, err := getInitConfigurationFromCluster(constants.KubernetesDir, client, newControlPlane, skipComponentConfigs) if err != nil { return nil, err } @@ -76,7 +76,7 @@ func FetchInitConfigurationFromCluster(client clientset.Interface, printer outpu } // getInitConfigurationFromCluster is separate only for testing purposes, don't call it directly, use FetchInitConfigurationFromCluster instead -func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Interface, getNodeRegistration, getAPIEndpoint, getComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { +func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Interface, newControlPlane, skipComponentConfigs bool) (*kubeadmapi.InitConfiguration, error) { // Also, the config map really should be KubeadmConfigConfigMap... configMap, err := apiclient.GetConfigMapWithShortRetry(client, metav1.NamespaceSystem, constants.KubeadmConfigConfigMap) if err != nil { @@ -106,28 +106,26 @@ func getInitConfigurationFromCluster(kubeconfigDir string, client clientset.Inte return nil, errors.Wrap(err, "failed to decode cluster configuration data") } - if getComponentConfigs { + if !skipComponentConfigs { // get the component configs from the corresponding config maps if err := componentconfigs.FetchFromCluster(&initcfg.ClusterConfiguration, client); err != nil { return nil, errors.Wrap(err, "failed to get component configs") } } - if getNodeRegistration { + // if this isn't a new controlplane instance (e.g. in case of kubeadm upgrades) + // get nodes specific information as well + if !newControlPlane { // gets the nodeRegistration for the current from the node object kubeconfigFile := filepath.Join(kubeconfigDir, constants.KubeletKubeConfigFileName) if err := GetNodeRegistration(kubeconfigFile, client, &initcfg.NodeRegistration, &initcfg.ClusterConfiguration); err != nil { return nil, errors.Wrap(err, "failed to get node registration") } - } - - if getAPIEndpoint { - // gets the APIEndpoint for the current control plane node - if err := GetAPIEndpoint(client, initcfg.NodeRegistration.Name, &initcfg.LocalAPIEndpoint); err != nil { + // gets the APIEndpoint for the current node + if err := getAPIEndpoint(client, initcfg.NodeRegistration.Name, &initcfg.LocalAPIEndpoint); err != nil { return nil, errors.Wrap(err, "failed to getAPIEndpoint") } } - return initcfg, nil } @@ -260,8 +258,7 @@ func getNodeNameFromSSR(client clientset.Interface) (string, error) { return strings.TrimPrefix(user, constants.NodesUserPrefix), nil } -// GetAPIEndpoint gets the API endpoint for a given node. -func GetAPIEndpoint(client clientset.Interface, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint) error { +func getAPIEndpoint(client clientset.Interface, nodeName string, apiEndpoint *kubeadmapi.APIEndpoint) error { return getAPIEndpointWithRetry(client, nodeName, apiEndpoint, constants.KubernetesAPICallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration) } diff --git a/cmd/kubeadm/app/util/config/cluster_test.go b/cmd/kubeadm/app/util/config/cluster_test.go index 41329eee4588b..94aa9917c93e0 100644 --- a/cmd/kubeadm/app/util/config/cluster_test.go +++ b/cmd/kubeadm/app/util/config/cluster_test.go @@ -491,14 +491,13 @@ func TestGetInitConfigurationFromCluster(t *testing.T) { defer os.RemoveAll(tmpdir) var tests = []struct { - name string - fileContents []byte - node *v1.Node - staticPods []testresources.FakeStaticPod - configMaps []testresources.FakeConfigMap - getNodeRegistration bool - getAPIEndpoint bool - expectedError bool + name string + fileContents []byte + node *v1.Node + staticPods []testresources.FakeStaticPod + configMaps []testresources.FakeConfigMap + newControlPlane bool + expectedError bool }{ { name: "invalid - No kubeadm-config ConfigMap", @@ -557,8 +556,6 @@ func TestGetInitConfigurationFromCluster(t *testing.T) { Taints: []v1.Taint{kubeadmconstants.ControlPlaneTaint}, }, }, - getNodeRegistration: true, - getAPIEndpoint: true, }, { name: "valid v1beta3 - new control plane == true", // InitConfiguration composed with data from different places, without node specific information @@ -591,8 +588,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) { }, }, }, - getNodeRegistration: false, - getAPIEndpoint: false, + newControlPlane: true, }, } @@ -633,8 +629,7 @@ func TestGetInitConfigurationFromCluster(t *testing.T) { } } - getComponentConfigs := true - cfg, err := getInitConfigurationFromCluster(tmpdir, client, rt.getNodeRegistration, rt.getAPIEndpoint, getComponentConfigs) + cfg, err := getInitConfigurationFromCluster(tmpdir, client, rt.newControlPlane, false) if rt.expectedError != (err != nil) { t.Errorf("unexpected return err from getInitConfigurationFromCluster: %v", err) return @@ -654,13 +649,13 @@ func TestGetInitConfigurationFromCluster(t *testing.T) { if cfg.NodeRegistration.ImagePullPolicy != kubeadmapiv1.DefaultImagePullPolicy { t.Errorf("invalid cfg.NodeRegistration.ImagePullPolicy %v", cfg.NodeRegistration.ImagePullPolicy) } - if rt.getNodeRegistration && rt.getAPIEndpoint && (cfg.LocalAPIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.LocalAPIEndpoint.BindPort != 1234) { + if !rt.newControlPlane && (cfg.LocalAPIEndpoint.AdvertiseAddress != "1.2.3.4" || cfg.LocalAPIEndpoint.BindPort != 1234) { t.Errorf("invalid cfg.LocalAPIEndpoint: %v", cfg.LocalAPIEndpoint) } - if rt.getNodeRegistration && (cfg.NodeRegistration.Name != nodeName || cfg.NodeRegistration.CRISocket != "myCRIsocket" || len(cfg.NodeRegistration.Taints) != 1) { + if !rt.newControlPlane && (cfg.NodeRegistration.Name != nodeName || cfg.NodeRegistration.CRISocket != "myCRIsocket" || len(cfg.NodeRegistration.Taints) != 1) { t.Errorf("invalid cfg.NodeRegistration: %v", cfg.NodeRegistration) } - if !rt.getNodeRegistration && len(cfg.NodeRegistration.CRISocket) > 0 { + if rt.newControlPlane && len(cfg.NodeRegistration.CRISocket) > 0 { t.Errorf("invalid cfg.NodeRegistration.CRISocket: expected empty CRISocket, but got %v", cfg.NodeRegistration.CRISocket) } if _, ok := cfg.ComponentConfigs[componentconfigs.KubeletGroup]; !ok { diff --git a/cmd/kubeadm/app/util/dryrun/dryrun.go b/cmd/kubeadm/app/util/dryrun/dryrun.go index a8ea57be902d2..2d1adff724447 100644 --- a/cmd/kubeadm/app/util/dryrun/dryrun.go +++ b/cmd/kubeadm/app/util/dryrun/dryrun.go @@ -19,10 +19,8 @@ package dryrun import ( "fmt" "io" - "net" "os" "path/filepath" - "strconv" "time" v1 "k8s.io/api/core/v1" @@ -105,11 +103,7 @@ func (w *Waiter) WaitForPodsWithLabel(kvLabel string) error { // WaitForKubelet blocks until the kubelet /healthz endpoint returns 'ok' func (w *Waiter) WaitForKubelet(healthzAddress string, healthzPort int32) error { - var ( - addrPort = net.JoinHostPort(healthzAddress, strconv.Itoa(int(healthzPort))) - healthzEndpoint = fmt.Sprintf("http://%s/healthz", addrPort) - ) - fmt.Printf("[dryrun] Would make sure the kubelet returns 'ok' at %s\n", healthzEndpoint) + fmt.Printf("[dryrun] Would make sure the kubelet returns 'ok' at http://%s:%d/healthz\n", healthzAddress, healthzPort) return nil } diff --git a/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go index 1b943645720d3..58bb616f50c8c 100644 --- a/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go +++ b/cmd/kubeadm/app/util/kubeconfig/kubeconfig.go @@ -104,21 +104,17 @@ func WriteToDisk(filename string, kubeconfig *clientcmdapi.Config) error { } // GetClusterFromKubeConfig returns the default Cluster of the specified KubeConfig -func GetClusterFromKubeConfig(config *clientcmdapi.Config) (string, *clientcmdapi.Cluster, error) { +func GetClusterFromKubeConfig(config *clientcmdapi.Config) (string, *clientcmdapi.Cluster) { // If there is an unnamed cluster object, use it if config.Clusters[""] != nil { - return "", config.Clusters[""], nil + return "", config.Clusters[""] } currentContext := config.Contexts[config.CurrentContext] if currentContext != nil { - if config.Clusters[currentContext.Cluster] != nil { - return currentContext.Cluster, config.Clusters[currentContext.Cluster], nil - } - return "", nil, errors.Errorf("no matching cluster for the current context: %s", config.CurrentContext) + return currentContext.Cluster, config.Clusters[currentContext.Cluster] } - - return "", nil, errors.Errorf("the current context is invalid: %s", config.CurrentContext) + return "", nil } // HasAuthenticationCredentials returns true if the current user has valid authentication credentials for diff --git a/cmd/kubeadm/app/util/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/util/kubeconfig/kubeconfig_test.go index d108d1acc292b..45934874b873d 100644 --- a/cmd/kubeadm/app/util/kubeconfig/kubeconfig_test.go +++ b/cmd/kubeadm/app/util/kubeconfig/kubeconfig_test.go @@ -351,53 +351,38 @@ func TestGetClusterFromKubeConfig(t *testing.T) { config *clientcmdapi.Config expectedClusterName string expectedCluster *clientcmdapi.Cluster - expectedError bool }{ { - name: "an existing cluster with an empty name is returned directly", + name: "cluster is empty", config: &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": {Server: "http://foo:8080"}, - }, - }, - expectedClusterName: "", - expectedCluster: &clientcmdapi.Cluster{ - Server: "http://foo:8080", - }, - }, - { - name: "the current context is invalid", - config: &clientcmdapi.Config{ - CurrentContext: "foo", - Contexts: map[string]*clientcmdapi.Context{ - "bar": {AuthInfo: "bar", Cluster: "bar"}, - }, + CurrentContext: "kubernetes", }, expectedClusterName: "", expectedCluster: nil, - expectedError: true, }, { - name: "no matching cluster for the current context", + name: "cluster and currentContext are not empty", config: &clientcmdapi.Config{ CurrentContext: "foo", Contexts: map[string]*clientcmdapi.Context{ - "foo": {AuthInfo: "bar", Cluster: "bar"}, + "foo": {AuthInfo: "foo", Cluster: "foo"}, + "bar": {AuthInfo: "bar", Cluster: "bar"}, }, Clusters: map[string]*clientcmdapi.Cluster{ - "baz": {Server: "https://bar:16443"}, + "foo": {Server: "http://foo:8080"}, + "bar": {Server: "https://bar:16443"}, }, }, - expectedClusterName: "", - expectedCluster: nil, - expectedError: true, + expectedClusterName: "foo", + expectedCluster: &clientcmdapi.Cluster{ + Server: "http://foo:8080", + }, }, { - name: "valid current context and cluster", + name: "cluster is not empty and currentContext is not in contexts", config: &clientcmdapi.Config{ CurrentContext: "foo", Contexts: map[string]*clientcmdapi.Context{ - "foo": {AuthInfo: "foo", Cluster: "foo"}, "bar": {AuthInfo: "bar", Cluster: "bar"}, }, Clusters: map[string]*clientcmdapi.Cluster{ @@ -405,25 +390,19 @@ func TestGetClusterFromKubeConfig(t *testing.T) { "bar": {Server: "https://bar:16443"}, }, }, - expectedClusterName: "foo", - expectedCluster: &clientcmdapi.Cluster{ - Server: "http://foo:8080", - }, + expectedClusterName: "", + expectedCluster: nil, }, } for _, rt := range tests { t.Run(rt.name, func(t *testing.T) { - clusterName, cluster, err := GetClusterFromKubeConfig(rt.config) + clusterName, cluster := GetClusterFromKubeConfig(rt.config) if clusterName != rt.expectedClusterName { t.Errorf("got cluster name = %s, expected %s", clusterName, rt.expectedClusterName) } if !reflect.DeepEqual(cluster, rt.expectedCluster) { t.Errorf("got cluster = %+v, expected %+v", cluster, rt.expectedCluster) } - if (err != nil) != rt.expectedError { - t.Errorf("expected error: %v, got: %v, error: %v", - rt.expectedError, err != nil, err) - } }) } } diff --git a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go index d733a6c6bcc6e..9fdde7be3ca6e 100644 --- a/cmd/kubeadm/app/util/pkiutil/pki_helpers.go +++ b/cmd/kubeadm/app/util/pkiutil/pki_helpers.go @@ -385,18 +385,15 @@ func GetAPIServerAltNames(cfg *kubeadmapi.InitConfiguration) (*certutil.AltNames return nil, errors.Wrapf(err, "unable to get first IP address from the given CIDR: %v", cfg.Networking.ServiceSubnet) } - var dnsNames []string - if len(cfg.NodeRegistration.Name) > 0 { - dnsNames = append(dnsNames, cfg.NodeRegistration.Name) - } - dnsNames = append(dnsNames, "kubernetes", "kubernetes.default", "kubernetes.default.svc") - if len(cfg.Networking.DNSDomain) > 0 { - dnsNames = append(dnsNames, fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain)) - } - // create AltNames with defaults DNSNames/IPs altNames := &certutil.AltNames{ - DNSNames: dnsNames, + DNSNames: []string{ + cfg.NodeRegistration.Name, + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + fmt.Sprintf("kubernetes.default.svc.%s", cfg.Networking.DNSDomain), + }, IPs: []net.IP{ internalAPIServerVirtualIP, advertiseAddress, @@ -444,16 +441,9 @@ func getAltNames(cfg *kubeadmapi.InitConfiguration, certName string) (*certutil. cfg.LocalAPIEndpoint.AdvertiseAddress) } - var dnsNames []string - if len(cfg.NodeRegistration.Name) > 0 { - dnsNames = []string{cfg.NodeRegistration.Name, "localhost"} - } else { - dnsNames = []string{"localhost"} - } - // create AltNames with defaults DNSNames/IPs altNames := &certutil.AltNames{ - DNSNames: dnsNames, + DNSNames: []string{cfg.NodeRegistration.Name, "localhost"}, IPs: []net.IP{advertiseAddress, net.IPv4(127, 0, 0, 1), net.IPv6loopback}, } @@ -675,15 +665,13 @@ func NewSelfSignedCACert(cfg *CertConfig, key crypto.Signer) (*x509.Certificate, CommonName: cfg.CommonName, Organization: cfg.Organization, }, + DNSNames: []string{cfg.CommonName}, NotBefore: notBefore, NotAfter: notAfter, KeyUsage: keyUsage, BasicConstraintsValid: true, IsCA: true, } - if len(cfg.CommonName) > 0 { - tmpl.DNSNames = []string{cfg.CommonName} - } certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key) if err != nil { diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go index b49940d6c9d09..5b70cb89c35b6 100644 --- a/cmd/kubeadm/app/util/staticpod/utils.go +++ b/cmd/kubeadm/app/util/staticpod/utils.go @@ -436,18 +436,3 @@ func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { hasher.Reset() fmt.Fprintf(hasher, "%v", dump.ForHash(objectToWrite)) } - -// IsControlPlaneNode returns true if the kube-apiserver static pod manifest is present -// on the host. -func IsControlPlaneNode() bool { - isControlPlaneNode := true - - filepath := kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.KubeAPIServer, - kubeadmconstants.GetStaticPodDirectory()) - - if _, err := os.Stat(filepath); os.IsNotExist(err) { - isControlPlaneNode = false - } - - return isControlPlaneNode -} diff --git a/go.mod b/go.mod index 1de91416f6f7b..b0f2492440880 100644 --- a/go.mod +++ b/go.mod @@ -124,9 +124,8 @@ require ( k8s.io/mount-utils v0.0.0 k8s.io/pod-security-admission v0.0.0 k8s.io/sample-apiserver v0.0.0 - k8s.io/system-validators v1.10.2 + k8s.io/system-validators v1.10.1 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 sigs.k8s.io/knftables v0.0.17 sigs.k8s.io/randfill v1.0.0 sigs.k8s.io/structured-merge-diff/v6 v6.3.0 @@ -229,6 +228,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/kustomize/api v0.20.1 // indirect sigs.k8s.io/kustomize/kustomize/v5 v5.7.1 // indirect sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect diff --git a/go.sum b/go.sum index 1dcfa9c29b8fe..c4622857565ca 100644 --- a/go.sum +++ b/go.sum @@ -548,8 +548,8 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/system-validators v1.10.2 h1:7rC7VdrQCaM55E08Pw3I1v1Op9ObLxdKAu5Ff5hIPwY= -k8s.io/system-validators v1.10.2/go.mod h1:awfSS706v9R12VC7u7K89FKfqVy44G+E0L1A0FX9Wmw= +k8s.io/system-validators v1.10.1 h1:bIO3YRgxJkh/W3ghcd5ViXNPGmjwQKlHk/ySPdw6K00= +k8s.io/system-validators v1.10.1/go.mod h1:awfSS706v9R12VC7u7K89FKfqVy44G+E0L1A0FX9Wmw= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= diff --git a/hack/lib/etcd.sh b/hack/lib/etcd.sh index 6e99d244c808b..c6fc45a5fab63 100755 --- a/hack/lib/etcd.sh +++ b/hack/lib/etcd.sh @@ -16,7 +16,7 @@ # A set of helpers for starting/running etcd for tests -ETCD_VERSION=${ETCD_VERSION:-3.6.5} +ETCD_VERSION=${ETCD_VERSION:-3.6.4} ETCD_HOST=${ETCD_HOST:-127.0.0.1} ETCD_PORT=${ETCD_PORT:-2379} # This is intentionally not called ETCD_LOG_LEVEL: diff --git a/hack/lib/util.sh b/hack/lib/util.sh index cae8a499204d1..6000505318ccf 100755 --- a/hack/lib/util.sh +++ b/hack/lib/util.sh @@ -478,7 +478,7 @@ function kube::util::create_client_certkey { done ${sudo} /usr/bin/env bash -e < 0 { warnings = append(warnings, "spec.externalIPs is ignored for headless services") } - if service.Spec.SessionAffinity != api.ServiceAffinityNone { + if service.Spec.SessionAffinity != "" { warnings = append(warnings, "spec.SessionAffinity is ignored for headless services") } } diff --git a/pkg/api/service/warnings_test.go b/pkg/api/service/warnings_test.go index dcf2a2ade19da..763d16f101cab 100644 --- a/pkg/api/service/warnings_test.go +++ b/pkg/api/service/warnings_test.go @@ -62,7 +62,6 @@ func TestGetWarningsForService(t *testing.T) { s.Spec.Type = api.ServiceTypeClusterIP s.Spec.ClusterIP = api.ClusterIPNone s.Spec.LoadBalancerIP = "1.2.3.4" - s.Spec.SessionAffinity = api.ServiceAffinityNone // default value }, numWarnings: 1, }, { @@ -71,11 +70,10 @@ func TestGetWarningsForService(t *testing.T) { s.Spec.Type = api.ServiceTypeClusterIP s.Spec.ClusterIP = api.ClusterIPNone s.Spec.ExternalIPs = []string{"1.2.3.4"} - s.Spec.SessionAffinity = api.ServiceAffinityNone // default value }, numWarnings: 1, }, { - name: "SessionAffinity Client IP set when headless service", + name: "SessionAffinity set when headless service", tweakSvc: func(s *api.Service) { s.Spec.Type = api.ServiceTypeClusterIP s.Spec.ClusterIP = api.ClusterIPNone @@ -83,25 +81,16 @@ func TestGetWarningsForService(t *testing.T) { }, numWarnings: 1, }, { - name: "SessionAffinity None set when headless service", + name: "ExternalIPs, LoadBalancerIP and SessionAffinity set when headless service", tweakSvc: func(s *api.Service) { s.Spec.Type = api.ServiceTypeClusterIP s.Spec.ClusterIP = api.ClusterIPNone - s.Spec.SessionAffinity = api.ServiceAffinityNone + s.Spec.ExternalIPs = []string{"1.2.3.4"} + s.Spec.LoadBalancerIP = "1.2.3.4" + s.Spec.SessionAffinity = api.ServiceAffinityClientIP }, - numWarnings: 0, - }, - { - name: "ExternalIPs, LoadBalancerIP and SessionAffinity set when headless service", - tweakSvc: func(s *api.Service) { - s.Spec.Type = api.ServiceTypeClusterIP - s.Spec.ClusterIP = api.ClusterIPNone - s.Spec.ExternalIPs = []string{"1.2.3.4"} - s.Spec.LoadBalancerIP = "1.2.3.4" - s.Spec.SessionAffinity = api.ServiceAffinityClientIP - }, - numWarnings: 3, - }} + numWarnings: 3, + }} for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/pkg/apis/batch/validation/validation.go b/pkg/apis/batch/validation/validation.go index 17ffbbe4b4b4a..5203c837ec514 100644 --- a/pkg/apis/batch/validation/validation.go +++ b/pkg/apis/batch/validation/validation.go @@ -913,15 +913,6 @@ func IsConditionTrue(list []batch.JobCondition, cType batch.JobConditionType) bo return false } -func IsConditionFalse(list []batch.JobCondition, cType batch.JobConditionType) bool { - for _, c := range list { - if c.Type == cType && c.Status == api.ConditionFalse { - return true - } - } - return false -} - func validateFailedIndexesNotOverlapCompleted(completedIndexesStr string, failedIndexesStr string, completions int32) error { if len(completedIndexesStr) == 0 || len(failedIndexesStr) == 0 { return nil diff --git a/pkg/apis/certificates/v1beta1/defaults_test.go b/pkg/apis/certificates/v1beta1/defaults_test.go index b2bf13f6fd409..fcfeef26ed33e 100644 --- a/pkg/apis/certificates/v1beta1/defaults_test.go +++ b/pkg/apis/certificates/v1beta1/defaults_test.go @@ -86,7 +86,7 @@ func TestIsKubeletServingCSR(t *testing.T) { exp: false, }, "does not default to kubelet-serving if it specifies an emailAddress SAN": { - req: newCSR(kubeletServerPEMOptions, pemOptions{emailAddresses: []string{"something@example.com"}}), + req: newCSR(kubeletServerPEMOptions, pemOptions{emailAddresses: []string{"something"}}), usages: kubeletServerUsages, exp: false, }, @@ -131,7 +131,7 @@ func TestIsKubeletClientCSR(t *testing.T) { exp: false, }, "does not default to kube-apiserver-client-kubelet if an emailAddress is set": { - req: newCSR(kubeletClientPEMOptions, pemOptions{emailAddresses: []string{"something@example.com"}}), + req: newCSR(kubeletClientPEMOptions, pemOptions{emailAddresses: []string{"something"}}), usages: kubeletClientUsages, exp: false, }, @@ -326,7 +326,7 @@ func TestSetDefaults_CertificateSigningRequestSpec_KubeletServing(t *testing.T) }, "does not default to kubelet-serving if it specifies an emailAddress SAN": { csr: capi.CertificateSigningRequestSpec{ - Request: csrWithOpts(kubeletServerPEMOptions, pemOptions{emailAddresses: []string{"something@example.com"}}), + Request: csrWithOpts(kubeletServerPEMOptions, pemOptions{emailAddresses: []string{"something"}}), Usages: kubeletServerUsages, Username: kubeletServerPEMOptions.cn, }, diff --git a/pkg/controller/controller_utils.go b/pkg/controller/controller_utils.go index c6c3b31f0d9ef..c847ad4f2295a 100644 --- a/pkg/controller/controller_utils.go +++ b/pkg/controller/controller_utils.go @@ -89,9 +89,12 @@ const ( // PodNodeNameKeyIndex is the name of the index used by PodInformer to index pods by their node name. PodNodeNameKeyIndex = "spec.nodeName" - // PodControllerIndex is the name for the Pod store's index function, - // which indexes by the key returned from PodControllerIndexKey. - PodControllerIndex = "podController" + // OrphanPodIndexKey is used to index all Orphan pods to this key + OrphanPodIndexKey = "_ORPHAN_POD" + + // podControllerUIDIndex is the name for the Pod store's index function, + // which is to index by pods's controllerUID. + PodControllerUIDIndex = "podControllerUID" ) var UpdateTaintBackoff = wait.Backoff{ @@ -1136,59 +1139,43 @@ func AddPodNodeNameIndexer(podInformer cache.SharedIndexInformer) error { }) } -// PodControllerIndexKey returns the index key to locate pods with the specified controller ownerReference. -// If ownerReference is nil, the returned key locates pods in the namespace without a controller ownerReference. -func PodControllerIndexKey(namespace string, ownerReference *metav1.OwnerReference) string { - if ownerReference == nil { - return namespace - } - return namespace + "/" + ownerReference.Kind + "/" + ownerReference.Name + "/" + string(ownerReference.UID) +// OrphanPodIndexKeyForNamespace returns the orphan pod index key for a specific namespace. +func OrphanPodIndexKeyForNamespace(namespace string) string { + return OrphanPodIndexKey + "/" + namespace } -// AddPodControllerIndexer adds an indexer for Pod's controllerRef.UID to the given PodInformer. +// AddPodControllerUIDIndexer adds an indexer for Pod's controllerRef.UID to the given PodInformer. // This indexer is used to efficiently look up pods by their ControllerRef.UID -func AddPodControllerIndexer(podInformer cache.SharedIndexInformer) error { - if _, exists := podInformer.GetIndexer().GetIndexers()[PodControllerIndex]; exists { +func AddPodControllerUIDIndexer(podInformer cache.SharedIndexInformer) error { + if _, exists := podInformer.GetIndexer().GetIndexers()[PodControllerUIDIndex]; exists { // indexer already exists, do nothing return nil } return podInformer.AddIndexers(cache.Indexers{ - PodControllerIndex: func(obj interface{}) ([]string, error) { + PodControllerUIDIndex: func(obj interface{}) ([]string, error) { pod, ok := obj.(*v1.Pod) if !ok { return nil, nil } - // Get the ControllerRef of the Pod to check if it's managed by a controller. - // Index with a non-nil controller (indicating an owned pod) or a nil controller (indicating an orphan pod). - return []string{PodControllerIndexKey(pod.Namespace, metav1.GetControllerOf(pod))}, nil + // Get the ControllerRef of the Pod to check if it's managed by a controller + if ref := metav1.GetControllerOf(pod); ref != nil { + return []string{string(ref.UID)}, nil + } + // If the Pod has no controller (i.e., it's orphaned), index it with the OrphanPodIndexKeyForNamespace + // This helps identify orphan pods for reconciliation and adoption by controllers + return []string{OrphanPodIndexKeyForNamespace(pod.Namespace)}, nil }, }) } // FilterPodsByOwner gets the Pods managed by an owner or orphan Pods in the owner's namespace -func FilterPodsByOwner(podIndexer cache.Indexer, owner *metav1.ObjectMeta, ownerKind string, includeOrphanedPods bool) ([]*v1.Pod, error) { +func FilterPodsByOwner(podIndexer cache.Indexer, owner *metav1.ObjectMeta) ([]*v1.Pod, error) { result := []*v1.Pod{} - - if len(owner.Namespace) == 0 { - return nil, fmt.Errorf("no owner namespace provided") - } - if len(owner.Name) == 0 { - return nil, fmt.Errorf("no owner name provided") - } - if len(owner.UID) == 0 { - return nil, fmt.Errorf("no owner uid provided") - } - if len(ownerKind) == 0 { - return nil, fmt.Errorf("no owner kind provided") - } - // Always include the owner key, which identifies Pods that are controlled by the owner - keys := []string{PodControllerIndexKey(owner.Namespace, &metav1.OwnerReference{Name: owner.Name, Kind: ownerKind, UID: owner.UID})} - if includeOrphanedPods { - // Optionally include the unowned key, which identifies orphaned Pods in the owner's namespace and might be adopted by the owner later - keys = append(keys, PodControllerIndexKey(owner.Namespace, nil)) - } - for _, key := range keys { - pods, err := podIndexer.ByIndex(PodControllerIndex, key) + // Iterate over two keys: + // - the UID of the owner, which identifies Pods that are controlled by the owner + // - the OrphanPodIndexKey, which identifies orphaned Pods in the owner's namespace and might be adopted by the owner later + for _, key := range []string{string(owner.UID), OrphanPodIndexKeyForNamespace(owner.Namespace)} { + pods, err := podIndexer.ByIndex(PodControllerUIDIndex, key) if err != nil { return nil, err } diff --git a/pkg/controller/controller_utils_test.go b/pkg/controller/controller_utils_test.go index 20fb8056418a3..819d44be1e5b1 100644 --- a/pkg/controller/controller_utils_test.go +++ b/pkg/controller/controller_utils_test.go @@ -1612,37 +1612,27 @@ func TestFilterPodsByOwner(t *testing.T) { return pod } - ownerKind := "OwnerKind" - ownerName := "ownerName" cases := map[string]struct { owner *metav1.ObjectMeta - ownedOnly bool allPods []*v1.Pod wantPodsKeys sets.Set[string] }{ "multiple Pods, some are owned by the owner": { owner: &metav1.ObjectMeta{ Namespace: "ns1", - Name: ownerName, UID: "abc", }, allPods: []*v1.Pod{ newPod("a", "ns1", &metav1.OwnerReference{ UID: "abc", - Kind: ownerKind, - Name: ownerName, Controller: ptr.To(true), }), newPod("b", "ns1", &metav1.OwnerReference{ UID: "def", - Kind: ownerKind, - Name: ownerName, Controller: ptr.To(true), }), newPod("c", "ns1", &metav1.OwnerReference{ UID: "abc", - Kind: ownerKind, - Name: ownerName, Controller: ptr.To(true), }), }, @@ -1651,8 +1641,6 @@ func TestFilterPodsByOwner(t *testing.T) { "orphan Pods in multiple namespaces": { owner: &metav1.ObjectMeta{ Namespace: "ns1", - Name: ownerName, - UID: "abc", }, allPods: []*v1.Pod{ newPod("a", "ns1", nil), @@ -1663,7 +1651,6 @@ func TestFilterPodsByOwner(t *testing.T) { "owned Pods and orphan Pods in the owner's namespace": { owner: &metav1.ObjectMeta{ Namespace: "ns1", - Name: ownerName, UID: "abc", }, allPods: []*v1.Pod{ @@ -1671,62 +1658,11 @@ func TestFilterPodsByOwner(t *testing.T) { newPod("b", "ns2", nil), newPod("c", "ns1", &metav1.OwnerReference{ UID: "abc", - Kind: ownerKind, - Name: ownerName, Controller: ptr.To(true), }), }, wantPodsKeys: sets.New("ns1/a", "ns1/c"), }, - "exclude orphan pods, pods in mismatched ns,uid,kind,name,controller": { - owner: &metav1.ObjectMeta{ - Namespace: "ns1", - Name: ownerName, - UID: "abc", - }, - allPods: []*v1.Pod{ - newPod("a", "ns1", nil), - newPod("other-ns-orphan", "ns2", nil), - newPod("other-ns-owned", "ns2", &metav1.OwnerReference{ - UID: "abc", - Kind: ownerKind, - Name: ownerName, - Controller: ptr.To(true), - }), - newPod("c", "ns1", &metav1.OwnerReference{ - UID: "abc", - Kind: ownerKind, - Name: ownerName, - Controller: ptr.To(true), - }), - newPod("other-uid", "ns1", &metav1.OwnerReference{ - UID: "other-uid", - Kind: ownerKind, - Name: ownerName, - Controller: ptr.To(true), - }), - newPod("other-kind", "ns1", &metav1.OwnerReference{ - UID: "abc", - Kind: "OtherKind", - Name: ownerName, - Controller: ptr.To(true), - }), - newPod("other-name", "ns1", &metav1.OwnerReference{ - UID: "abc", - Kind: ownerKind, - Name: "otherName", - Controller: ptr.To(true), - }), - newPod("non-controller", "ns1", &metav1.OwnerReference{ - UID: "abc", - Kind: ownerKind, - Name: ownerName, - Controller: ptr.To(false), - }), - }, - ownedOnly: true, - wantPodsKeys: sets.New("ns1/c"), - }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { @@ -1734,7 +1670,7 @@ func TestFilterPodsByOwner(t *testing.T) { sharedInformers := informers.NewSharedInformerFactory(fakeClient, 0) podInformer := sharedInformers.Core().V1().Pods() - err := AddPodControllerIndexer(podInformer.Informer()) + err := AddPodControllerUIDIndexer(podInformer.Informer()) if err != nil { t.Fatalf("failed to register indexer: %v", err) } @@ -1744,10 +1680,7 @@ func TestFilterPodsByOwner(t *testing.T) { t.Fatalf("failed adding Pod to indexer: %v", err) } } - gotPods, err := FilterPodsByOwner(podIndexer, tc.owner, ownerKind, !tc.ownedOnly) - if err != nil { - t.Fatal(err) - } + gotPods, _ := FilterPodsByOwner(podIndexer, tc.owner) gotPodKeys := sets.New[string]() for _, pod := range gotPods { gotPodKeys.Insert(pod.Namespace + "/" + pod.Name) diff --git a/pkg/controller/daemon/daemon_controller.go b/pkg/controller/daemon/daemon_controller.go index 0ab4290ae55cf..4965cb75c8a30 100644 --- a/pkg/controller/daemon/daemon_controller.go +++ b/pkg/controller/daemon/daemon_controller.go @@ -223,7 +223,7 @@ func NewDaemonSetsController( dsc.podLister = podInformer.Lister() dsc.podStoreSynced = podInformer.Informer().HasSynced controller.AddPodNodeNameIndexer(podInformer.Informer()) - controller.AddPodControllerIndexer(podInformer.Informer()) + controller.AddPodControllerUIDIndexer(podInformer.Informer()) dsc.podIndexer = podInformer.Informer().GetIndexer() nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -710,7 +710,7 @@ func (dsc *DaemonSetsController) getDaemonPods(ctx context.Context, ds *apps.Dae return nil, err } // List all pods indexed to DS UID and Orphan pods - pods, err := controller.FilterPodsByOwner(dsc.podIndexer, &ds.ObjectMeta, "DaemonSet", true) + pods, err := controller.FilterPodsByOwner(dsc.podIndexer, &ds.ObjectMeta) if err != nil { return nil, err } diff --git a/pkg/controller/history/controller_history.go b/pkg/controller/history/controller_history.go index 9f4f82223a19a..19ae0999af543 100644 --- a/pkg/controller/history/controller_history.go +++ b/pkg/controller/history/controller_history.go @@ -111,9 +111,27 @@ func SortControllerRevisions(revisions []*apps.ControllerRevision) { // EqualRevision returns true if lhs and rhs are either both nil, or both point to non-nil ControllerRevisions that // contain semantically equivalent data. Otherwise this method returns false. func EqualRevision(lhs *apps.ControllerRevision, rhs *apps.ControllerRevision) bool { + var lhsHash, rhsHash *uint32 if lhs == nil || rhs == nil { return lhs == rhs } + if hs, found := lhs.Labels[ControllerRevisionHashLabel]; found { + hash, err := strconv.ParseInt(hs, 10, 32) + if err == nil { + lhsHash = new(uint32) + *lhsHash = uint32(hash) + } + } + if hs, found := rhs.Labels[ControllerRevisionHashLabel]; found { + hash, err := strconv.ParseInt(hs, 10, 32) + if err == nil { + rhsHash = new(uint32) + *rhsHash = uint32(hash) + } + } + if lhsHash != nil && rhsHash != nil && *lhsHash != *rhsHash { + return false + } return bytes.Equal(lhs.Data.Raw, rhs.Data.Raw) && apiequality.Semantic.DeepEqual(lhs.Data.Object, rhs.Data.Object) } diff --git a/pkg/controller/job/job_controller.go b/pkg/controller/job/job_controller.go index 4de4bdc302a59..6a74ee0bd53a9 100644 --- a/pkg/controller/job/job_controller.go +++ b/pkg/controller/job/job_controller.go @@ -226,7 +226,7 @@ func newControllerWithClock(ctx context.Context, podInformer coreinformers.PodIn jm.podStore = podInformer.Lister() jm.podStoreSynced = podInformer.Informer().HasSynced - err := controller.AddPodControllerIndexer(podInformer.Informer()) + err := controller.AddPodControllerUIDIndexer(podInformer.Informer()) if err != nil { return nil, fmt.Errorf("adding Pod controller UID indexer: %w", err) } @@ -769,7 +769,7 @@ func (jm *Controller) getPodsForJob(ctx context.Context, j *batch.Job) ([]*v1.Po } // list all pods managed by this Job using the pod indexer - pods, err := controller.FilterPodsByOwner(jm.podIndexer, &j.ObjectMeta, "Job", true) + pods, err := controller.FilterPodsByOwner(jm.podIndexer, &j.ObjectMeta) if err != nil { return nil, err } diff --git a/pkg/controller/job/job_controller_test.go b/pkg/controller/job/job_controller_test.go index b2970f94149a1..891feef67a33d 100644 --- a/pkg/controller/job/job_controller_test.go +++ b/pkg/controller/job/job_controller_test.go @@ -28,7 +28,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - batch "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -2950,7 +2949,6 @@ func TestSyncJobWhenManagedBy(t *testing.T) { TypeMeta: metav1.TypeMeta{Kind: "Job"}, ObjectMeta: metav1.ObjectMeta{ Name: "foobar", - UID: "foobaruid", Namespace: metav1.NamespaceDefault, }, Spec: batch.JobSpec{ diff --git a/pkg/controller/replicaset/replica_set.go b/pkg/controller/replicaset/replica_set.go index 204f4c46a6dd8..8934bd4605500 100644 --- a/pkg/controller/replicaset/replica_set.go +++ b/pkg/controller/replicaset/replica_set.go @@ -220,7 +220,7 @@ func NewBaseController(logger klog.Logger, rsInformer appsinformers.ReplicaSetIn }) rsc.podLister = podInformer.Lister() rsc.podListerSynced = podInformer.Informer().HasSynced - controller.AddPodControllerIndexer(podInformer.Informer()) //nolint:errcheck + controller.AddPodControllerUIDIndexer(podInformer.Informer()) //nolint:errcheck rsc.podIndexer = podInformer.Informer().GetIndexer() rsc.syncHandler = rsc.syncReplicaSet @@ -728,7 +728,7 @@ func (rsc *ReplicaSetController) syncReplicaSet(ctx context.Context, key string) } // List all pods indexed to RS UID and Orphan pods - allRSPods, err := controller.FilterPodsByOwner(rsc.podIndexer, &rs.ObjectMeta, rsc.Kind, true) + allRSPods, err := controller.FilterPodsByOwner(rsc.podIndexer, &rs.ObjectMeta) if err != nil { return err } diff --git a/pkg/controller/statefulset/stateful_set.go b/pkg/controller/statefulset/stateful_set.go index 007766df25d5c..36e49ca0844d3 100644 --- a/pkg/controller/statefulset/stateful_set.go +++ b/pkg/controller/statefulset/stateful_set.go @@ -131,7 +131,7 @@ func NewStatefulSetController( }) ssc.podLister = podInformer.Lister() ssc.podListerSynced = podInformer.Informer().HasSynced - controller.AddPodControllerIndexer(podInformer.Informer()) + controller.AddPodControllerUIDIndexer(podInformer.Informer()) ssc.podIndexer = podInformer.Informer().GetIndexer() setInformer.Informer().AddEventHandler( cache.ResourceEventHandlerFuncs{ @@ -312,7 +312,7 @@ func (ssc *StatefulSetController) deletePod(logger klog.Logger, obj interface{}) // NOTE: Returned Pods are pointers to objects from the cache. // If you need to modify one, you need to copy it first. func (ssc *StatefulSetController) getPodsForStatefulSet(ctx context.Context, set *apps.StatefulSet, selector labels.Selector) ([]*v1.Pod, error) { - podsForSts, err := controller.FilterPodsByOwner(ssc.podIndexer, &set.ObjectMeta, "StatefulSet", true) + podsForSts, err := controller.FilterPodsByOwner(ssc.podIndexer, &set.ObjectMeta) if err != nil { return nil, err } diff --git a/pkg/controller/statefulset/stateful_set_compatibility_test.go b/pkg/controller/statefulset/stateful_set_compatibility_test.go deleted file mode 100644 index ce3d3dcb1bdc1..0000000000000 --- a/pkg/controller/statefulset/stateful_set_compatibility_test.go +++ /dev/null @@ -1,149 +0,0 @@ -/* -Copyright 2025 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package statefulset - -import ( - "os" - "reflect" - "testing" - - "github.com/google/go-cmp/cmp" - "sigs.k8s.io/json" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/fake" - "k8s.io/kubernetes/pkg/api/legacyscheme" -) - -func TestStatefulSetCompatibility(t *testing.T) { - set133 := &appsv1.StatefulSet{} - set134 := &appsv1.StatefulSet{} - rev133 := &appsv1.ControllerRevision{} - rev134 := &appsv1.ControllerRevision{} - load(t, "compatibility_set_1.33.0.json", set133) - load(t, "compatibility_set_1.34.0.json", set134) - load(t, "compatibility_revision_1.33.0.json", rev133) - load(t, "compatibility_revision_1.34.0.json", rev134) - - testcases := []struct { - name string - set *appsv1.StatefulSet - revisions []*appsv1.ControllerRevision - }{ - { - name: "1.33 set, 1.33 rev", - set: set133.DeepCopy(), - revisions: []*appsv1.ControllerRevision{rev133.DeepCopy()}, - }, - { - name: "1.34 set, 1.34 rev", - set: set134.DeepCopy(), - revisions: []*appsv1.ControllerRevision{rev134.DeepCopy()}, - }, - { - name: "1.34 set, 1.33+1.34 rev", - set: set134.DeepCopy(), - revisions: []*appsv1.ControllerRevision{rev133.DeepCopy(), rev134.DeepCopy()}, - }, - } - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - latestRev := tc.revisions[len(tc.revisions)-1] - client := fake.NewClientset(tc.set) - _, _, ssc := setupController(client) - currentRev, updateRev, _, err := ssc.(*defaultStatefulSetControl).getStatefulSetRevisions(tc.set, tc.revisions) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(currentRev, latestRev) { - t.Fatalf("expected no change from latestRev, got %s", cmp.Diff(latestRev, currentRev)) - } - if !reflect.DeepEqual(updateRev, latestRev) { - t.Fatalf("expected no change from latestRev, got %s", cmp.Diff(latestRev, updateRev)) - } - }) - } -} - -func BenchmarkStatefulSetCompatibility(b *testing.B) { - set133 := &appsv1.StatefulSet{} - set134 := &appsv1.StatefulSet{} - rev133 := &appsv1.ControllerRevision{} - rev134 := &appsv1.ControllerRevision{} - load(b, "compatibility_set_1.33.0.json", set133) - load(b, "compatibility_set_1.34.0.json", set134) - load(b, "compatibility_revision_1.33.0.json", rev133) - load(b, "compatibility_revision_1.34.0.json", rev134) - - testcases := []struct { - name string - set *appsv1.StatefulSet - revisions []*appsv1.ControllerRevision - }{ - { - name: "1.33 set, 1.33 rev", - set: set133.DeepCopy(), - revisions: []*appsv1.ControllerRevision{rev133.DeepCopy()}, - }, - { - name: "1.34 set, 1.34 rev", - set: set134.DeepCopy(), - revisions: []*appsv1.ControllerRevision{rev134.DeepCopy()}, - }, - { - name: "1.34 set, 1.33+1.34 rev", - set: set134.DeepCopy(), - revisions: []*appsv1.ControllerRevision{rev133.DeepCopy(), rev134.DeepCopy()}, - }, - } - - for _, tc := range testcases { - b.Run(tc.name, func(b *testing.B) { - latestRev := tc.revisions[len(tc.revisions)-1] - client := fake.NewClientset(tc.set) - _, _, ssc := setupController(client) - for i := 0; i < b.N; i++ { - currentRev, updateRev, _, err := ssc.(*defaultStatefulSetControl).getStatefulSetRevisions(tc.set, tc.revisions) - if err != nil { - b.Fatal(err) - } - if !reflect.DeepEqual(currentRev, latestRev) { - b.Fatalf("expected no change from latestRev, got %s", cmp.Diff(latestRev, currentRev)) - } - if !reflect.DeepEqual(updateRev, latestRev) { - b.Fatalf("expected no change from latestRev, got %s", cmp.Diff(latestRev, updateRev)) - } - } - }) - } -} - -func load(t testing.TB, filename string, object runtime.Object) { - data, err := os.ReadFile("testdata/" + filename) - if err != nil { - t.Fatal(err) - } - if strictErrs, err := json.UnmarshalStrict(data, object); err != nil { - t.Fatal(err) - } else if len(strictErrs) > 0 { - t.Fatal(strictErrs) - } - // apply defaulting just as if it was read from etcd - legacyscheme.Scheme.Default(object) -} diff --git a/pkg/controller/statefulset/stateful_set_control.go b/pkg/controller/statefulset/stateful_set_control.go index 78dd40081e5ea..7d4f4990ba76c 100644 --- a/pkg/controller/statefulset/stateful_set_control.go +++ b/pkg/controller/statefulset/stateful_set_control.go @@ -21,17 +21,13 @@ import ( "sort" "sync" - "k8s.io/klog/v2" - "k8s.io/utils/lru" - apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/kubernetes/pkg/api/legacyscheme" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/controller/history" "k8s.io/kubernetes/pkg/features" ) @@ -65,15 +61,13 @@ func NewDefaultStatefulSetControl( podControl *StatefulPodControl, statusUpdater StatefulSetStatusUpdaterInterface, controllerHistory history.Interface) StatefulSetControlInterface { - return &defaultStatefulSetControl{podControl, statusUpdater, controllerHistory, lru.New(maxRevisionEqualityCacheEntries)} + return &defaultStatefulSetControl{podControl, statusUpdater, controllerHistory} } type defaultStatefulSetControl struct { podControl *StatefulPodControl statusUpdater StatefulSetStatusUpdaterInterface controllerHistory history.Interface - - revisionEqualityCache *lru.Cache } // UpdateStatefulSet executes the core logic loop for a stateful set, applying the predictable and @@ -213,49 +207,6 @@ func (ssc *defaultStatefulSetControl) truncateHistory( return nil } -// maxRevisionEqualityCacheEntries is the size of the memory cache for equal set/controllerrevisions. -// Allowing up to 10,000 entries takes ~1MB. Each entry consumes up to ~111 bytes: -// - 40 bytes for the cache key (revisionEqualityKey{}) -// - 16 for the cache value (interface{} --> struct{}{}) -// - 36 bytes for the setUID string -// - 19 bytes for the revisionResourceVersion string -const maxRevisionEqualityCacheEntries = 10_000 - -// revisionEqualityKey is the cache key for remembering a particular revision RV -// is equal to the revision that results from a particular set UID at a particular set generation. -type revisionEqualityKey struct { - setUID types.UID - setGeneration int64 - revisionResourceVersion string -} - -// setMatchesLatestExistingRevision returns true if the set/proposedRevision already matches what would be produced from restoring latestExistingRevision. -func setMatchesLatestExistingRevision(set *apps.StatefulSet, proposedRevision *apps.ControllerRevision, latestExistingRevision *apps.ControllerRevision, memory *lru.Cache) bool { - if !utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetSemanticRevisionComparison) { - return false - } - equalityCacheKey := revisionEqualityKey{setUID: set.UID, setGeneration: set.Generation, revisionResourceVersion: latestExistingRevision.ResourceVersion} - if _, ok := memory.Get(equalityCacheKey); ok { - return true - } - // see if reverting to the latest existing revision would produce the same thing as proposedRevision - latestSet, err := ApplyRevision(set, latestExistingRevision) - if err != nil { - return false - } - legacyscheme.Scheme.Default(latestSet) - reconstructedLatestRevision, err := newRevision(latestSet, -1, nil) - if err != nil { - return false - } - // if they match, cache this combination of set(uid,generation)+revision(resourceVersion) to minimize expensive comparisons in steady state - if history.EqualRevision(proposedRevision, reconstructedLatestRevision) { - memory.Add(equalityCacheKey, struct{}{}) - return true - } - return false -} - // getStatefulSetRevisions returns the current and update ControllerRevisions for set. It also // returns a collision count that records the number of name collisions set saw when creating // new ControllerRevisions. This count is incremented on every name collision and is used in @@ -299,9 +250,6 @@ func (ssc *defaultStatefulSetControl) getStatefulSetRevisions( if err != nil { return nil, nil, collisionCount, err } - } else if revisionCount > 0 && setMatchesLatestExistingRevision(set, updateRevision, revisions[revisionCount-1], ssc.revisionEqualityCache) { - // the update revision has not changed - updateRevision = revisions[revisionCount-1] } else { //if there is no equivalent revision we create a new one updateRevision, err = ssc.controllerHistory.CreateControllerRevision(set, updateRevision, &collisionCount) diff --git a/pkg/controller/statefulset/stateful_set_control_test.go b/pkg/controller/statefulset/stateful_set_control_test.go index a1c6356f52b80..897ec9c2976ec 100644 --- a/pkg/controller/statefulset/stateful_set_control_test.go +++ b/pkg/controller/statefulset/stateful_set_control_test.go @@ -31,7 +31,6 @@ import ( "testing" "time" - "k8s.io/utils/lru" "k8s.io/utils/ptr" apps "k8s.io/api/apps/v1" @@ -884,7 +883,7 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) { informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc()) spc := NewStatefulPodControlFromManager(newFakeObjectManager(informerFactory), &noopRecorder{}) ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets()) - ssc := defaultStatefulSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), lru.New(maxRevisionEqualityCacheEntries)} + ssc := defaultStatefulSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions())} stop := make(chan struct{}) defer close(stop) diff --git a/pkg/controller/statefulset/testdata/compatibility_revision_1.33.0.json b/pkg/controller/statefulset/testdata/compatibility_revision_1.33.0.json deleted file mode 100644 index 0b874adf4c3cc..0000000000000 --- a/pkg/controller/statefulset/testdata/compatibility_revision_1.33.0.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "apiVersion":"apps/v1", - "kind":"ControllerRevision", - "metadata":{ - "creationTimestamp":"2025-10-31T18:19:02Z", - "labels":{ - "app":"foo", - "controller.kubernetes.io/hash":"c77f6d978" - }, - "name":"test-c77f6d978", - "namespace":"default", - "ownerReferences":[{ - "apiVersion":"apps/v1", - "blockOwnerDeletion":true, - "controller":true, - "kind":"StatefulSet", - "name":"test", - "uid":"ec335e25-1045-4216-8634-50cfbe05f3d6" - }], - "resourceVersion":"2209", - "uid":"af6e1945-ed14-4d1a-b420-813aa683a0fd" - }, - "data":{"spec":{"template":{"$patch":"replace","metadata":{"annotations":{"test":"value"},"creationTimestamp":null,"labels":{"app":"foo"}},"spec":{"containers":[{"image":"test","imagePullPolicy":"Always","name":"test","resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}}}, - "revision":1 -} diff --git a/pkg/controller/statefulset/testdata/compatibility_revision_1.34.0.json b/pkg/controller/statefulset/testdata/compatibility_revision_1.34.0.json deleted file mode 100644 index 085eea4da0203..0000000000000 --- a/pkg/controller/statefulset/testdata/compatibility_revision_1.34.0.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "apiVersion":"apps/v1", - "kind":"ControllerRevision", - "metadata":{ - "creationTimestamp":"2025-11-03T19:46:23Z", - "labels":{ - "app":"foo", - "controller.kubernetes.io/hash":"776999688b" - }, - "name":"test-776999688b", - "namespace":"default", - "ownerReferences":[{ - "apiVersion":"apps/v1", - "blockOwnerDeletion":true, - "controller":true, - "kind":"StatefulSet", - "name":"test", - "uid":"ec335e25-1045-4216-8634-50cfbe05f3d6" - }], - "resourceVersion":"16318", - "uid":"47df387b-5f17-40b6-9964-4c43cf6ad5d1" - }, - "data":{"spec":{"template":{"$patch":"replace","metadata":{"annotations":{"test":"value"},"labels":{"app":"foo"}},"spec":{"containers":[{"image":"test","imagePullPolicy":"Always","name":"test","resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}}}, - "revision":2 -} diff --git a/pkg/controller/statefulset/testdata/compatibility_set_1.33.0.json b/pkg/controller/statefulset/testdata/compatibility_set_1.33.0.json deleted file mode 100644 index e263ffad12314..0000000000000 --- a/pkg/controller/statefulset/testdata/compatibility_set_1.33.0.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "apiVersion": "apps/v1", - "kind": "StatefulSet", - "metadata": { - "creationTimestamp": "2025-10-31T18:19:02Z", - "generation": 1, - "labels": { - "sslabel": "value" - }, - "name": "test", - "namespace": "default", - "resourceVersion": "2219", - "uid": "ec335e25-1045-4216-8634-50cfbe05f3d6" - }, - "spec": { - "persistentVolumeClaimRetentionPolicy": { - "whenDeleted": "Retain", - "whenScaled": "Retain" - }, - "podManagementPolicy": "OrderedReady", - "replicas": 1, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "app": "foo" - } - }, - "serviceName": "", - "template": { - "metadata": { - "annotations": { - "test": "value" - }, - "creationTimestamp": null, - "labels": { - "app": "foo" - } - }, - "spec": { - "containers": [ - { - "image": "test", - "imagePullPolicy": "Always", - "name": "test", - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File" - } - ], - "dnsPolicy": "ClusterFirst", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "terminationGracePeriodSeconds": 30 - } - }, - "updateStrategy": { - "rollingUpdate": { - "partition": 0 - }, - "type": "RollingUpdate" - }, - "volumeClaimTemplates": [ - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "metadata": { - "annotations": { - "key": "value" - }, - "creationTimestamp": null, - "labels": { - "key": "value" - }, - "name": "test" - }, - "spec": { - "accessModes": [ - "ReadWriteOnce" - ], - "resources": { - "requests": { - "storage": "1Gi" - } - }, - "volumeMode": "Filesystem" - }, - "status": { - "phase": "Pending" - } - } - ] - }, - "status": { - "availableReplicas": 1, - "collisionCount": 0, - "currentReplicas": 1, - "currentRevision": "test-c77f6d978", - "observedGeneration": 1, - "replicas": 1, - "updateRevision": "test-c77f6d978", - "updatedReplicas": 1 - } -} \ No newline at end of file diff --git a/pkg/controller/statefulset/testdata/compatibility_set_1.34.0.json b/pkg/controller/statefulset/testdata/compatibility_set_1.34.0.json deleted file mode 100644 index 45a3fc56ec4b0..0000000000000 --- a/pkg/controller/statefulset/testdata/compatibility_set_1.34.0.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "apiVersion": "apps/v1", - "kind": "StatefulSet", - "metadata": { - "creationTimestamp": "2025-10-31T18:19:02Z", - "generation": 1, - "labels": { - "sslabel": "value" - }, - "name": "test", - "namespace": "default", - "resourceVersion": "16319", - "uid": "ec335e25-1045-4216-8634-50cfbe05f3d6" - }, - "spec": { - "persistentVolumeClaimRetentionPolicy": { - "whenDeleted": "Retain", - "whenScaled": "Retain" - }, - "podManagementPolicy": "OrderedReady", - "replicas": 1, - "revisionHistoryLimit": 10, - "selector": { - "matchLabels": { - "app": "foo" - } - }, - "serviceName": "", - "template": { - "metadata": { - "annotations": { - "test": "value" - }, - "labels": { - "app": "foo" - } - }, - "spec": { - "containers": [ - { - "image": "test", - "imagePullPolicy": "Always", - "name": "test", - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File" - } - ], - "dnsPolicy": "ClusterFirst", - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "terminationGracePeriodSeconds": 30 - } - }, - "updateStrategy": { - "rollingUpdate": { - "partition": 0 - }, - "type": "RollingUpdate" - }, - "volumeClaimTemplates": [ - { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "metadata": { - "annotations": { - "key": "value" - }, - "labels": { - "key": "value" - }, - "name": "test" - }, - "spec": { - "accessModes": [ - "ReadWriteOnce" - ], - "resources": { - "requests": { - "storage": "1Gi" - } - }, - "volumeMode": "Filesystem" - }, - "status": { - "phase": "Pending" - } - } - ] - }, - "status": { - "availableReplicas": 1, - "collisionCount": 0, - "currentReplicas": 1, - "currentRevision": "test-776999688b", - "observedGeneration": 1, - "replicas": 1, - "updateRevision": "test-776999688b", - "updatedReplicas": 1 - } -} \ No newline at end of file diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 5b7794431e557..a3b55db585a3c 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -941,12 +941,6 @@ const ( // Enables policies controlling deletion of PVCs created by a StatefulSet. StatefulSetAutoDeletePVC featuregate.Feature = "StatefulSetAutoDeletePVC" - // owner: @liggitt - // - // Mitigates spurious statefulset rollouts due to controller revision comparison mismatches - // which are not semantically significant (e.g. serialization differences or missing defaulted fields). - StatefulSetSemanticRevisionComparison = "StatefulSetSemanticRevisionComparison" - // owner: @cupnes // kep: https://kep.k8s.io/4049 // @@ -1685,7 +1679,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate }, SchedulerAsyncAPICalls: { - {Version: version.MustParse("1.34"), Default: false, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.Beta}, }, SchedulerAsyncPreemption: { @@ -1761,12 +1755,6 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.32"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // GA in 1.32, remove in 1.35 }, - StatefulSetSemanticRevisionComparison: { - // This is a mitigation for a 1.34 regression due to serialization differences that cannot be feature-gated, - // so this mitigation should not auto-disable even if emulating versions prior to 1.34 with --emulation-version. - {Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.Beta}, - }, - StorageCapacityScoring: { {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, }, diff --git a/pkg/kubelet/cm/dra/plugin/dra_plugin_manager.go b/pkg/kubelet/cm/dra/plugin/dra_plugin_manager.go index 5305546ccf969..2b76d3bd34816 100644 --- a/pkg/kubelet/cm/dra/plugin/dra_plugin_manager.go +++ b/pkg/kubelet/cm/dra/plugin/dra_plugin_manager.go @@ -62,9 +62,6 @@ type DRAPluginManager struct { wipingDelay time.Duration streamHandler StreamHandler - // withIdleTimeout is only for unit testing, ignore if <= 0. - withIdleTimeout time.Duration - wg sync.WaitGroup mutex sync.RWMutex @@ -118,13 +115,7 @@ func (m *monitoredPlugin) HandleConn(_ context.Context, stats grpcstats.ConnStat case *grpcstats.ConnEnd: // We have to ask for a reconnect, otherwise gRPC wouldn't try and // thus we wouldn't be notified about a restart of the plugin. - // - // This must be done in a goroutine because gRPC deadlocks - // when called directly from inside HandleConn when a connection - // goes idle (and only then). It looks like cc.idlenessMgr.ExitIdleMode - // in Connect tries to lock a mutex that is already locked by - // the caller of HandleConn. - go m.conn.Connect() + m.conn.Connect() default: return } @@ -370,15 +361,12 @@ func (pm *DRAPluginManager) add(driverName string, endpoint string, chosenServic // The gRPC connection gets created once. gRPC then connects to the gRPC server on demand. target := "unix:" + endpoint logger.V(4).Info("Creating new gRPC connection", "target", target) - options := []grpc.DialOption{ + conn, err := grpc.NewClient( + target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithChainUnaryInterceptor(newMetricsInterceptor(driverName)), grpc.WithStatsHandler(mp), - } - if pm.withIdleTimeout > 0 { - options = append(options, grpc.WithIdleTimeout(pm.withIdleTimeout)) - } - conn, err := grpc.NewClient(target, options...) + ) if err != nil { return fmt.Errorf("create gRPC connection to DRA driver %s plugin at endpoint %s: %w", driverName, endpoint, err) } diff --git a/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go b/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go index c8eb975b9bbff..b3af8c75b32de 100644 --- a/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go +++ b/pkg/kubelet/cm/dra/plugin/dra_plugin_test.go @@ -185,44 +185,6 @@ func TestGRPCConnIsReused(t *testing.T) { require.Equal(t, 2, reusedConns[conn], "expected counter to be 2 but got %d", reusedConns[conn]) } -func TestGRPCConnUsableAfterIdle(t *testing.T) { - tCtx := ktesting.Init(t) - service := drapbv1.DRAPluginService - addr := path.Join(t.TempDir(), "dra.sock") - teardown, err := setupFakeGRPCServer(service, addr) - require.NoError(t, err) - defer teardown() - - driverName := "dummy-driver" - - // ensure the plugin we are using is registered - draPlugins := NewDRAPluginManager(tCtx, nil, nil, &mockStreamHandler{}, 0) - draPlugins.withIdleTimeout = 5 * time.Second - tCtx.ExpectNoError(draPlugins.add(driverName, addr, service, defaultClientCallTimeout), "add plugin") - plugin, err := draPlugins.GetPlugin(driverName) - tCtx.ExpectNoError(err, "get plugin") - - // The connection doesn't really become idle because HandleConn - // kicks it back to ready by calling Connect. Just sleep long - // enough here, the code should be reached... - tCtx.Log("Waiting for idle timeout...") - time.Sleep(2 * draPlugins.withIdleTimeout) - - req := &drapbv1.NodePrepareResourcesRequest{ - Claims: []*drapbv1.Claim{ - { - Namespace: "dummy-namespace", - UID: "dummy-uid", - Name: "dummy-claim", - }, - }, - } - - callCtx := ktesting.WithTimeout(tCtx, 10*time.Second, "call timed out") - _, err = plugin.NodePrepareResources(callCtx, req) - tCtx.ExpectNoError(err, "NodePrepareResources") -} - func TestGetDRAPlugin(t *testing.T) { for _, test := range []struct { description string diff --git a/pkg/kubelet/prober/worker.go b/pkg/kubelet/prober/worker.go index 60461165bdafd..8578c415d4483 100644 --- a/pkg/kubelet/prober/worker.go +++ b/pkg/kubelet/prober/worker.go @@ -83,16 +83,6 @@ type worker struct { proberDurationUnknownMetricLabels metrics.Labels } -// isInitContainer checks if the worker's container is in the pod's init containers -func (w *worker) isInitContainer() bool { - for _, initContainer := range w.pod.Spec.InitContainers { - if initContainer.Name == w.container.Name { - return true - } - } - return false -} - // Creates and starts a new probe worker. func newWorker( m *manager, @@ -263,17 +253,12 @@ func (w *worker) doProbe(ctx context.Context) (keepGoing bool) { if !w.containerID.IsEmpty() { w.resultsManager.Set(w.containerID, results.Failure, w.pod) } - - isRestartableInitContainer := w.isInitContainer() && - w.container.RestartPolicy != nil && *w.container.RestartPolicy == v1.ContainerRestartPolicyAlways - // Abort if the container will not be restarted. if utilfeature.DefaultFeatureGate.Enabled(features.ContainerRestartRules) { return c.State.Terminated != nil || podutil.IsContainerRestartable(w.pod.Spec, w.container) } return c.State.Terminated == nil || - w.pod.Spec.RestartPolicy != v1.RestartPolicyNever || - isRestartableInitContainer + w.pod.Spec.RestartPolicy != v1.RestartPolicyNever } // Graceful shutdown of the pod. diff --git a/pkg/kubelet/prober/worker_test.go b/pkg/kubelet/prober/worker_test.go index 946f183528642..ec8f8673ec5f5 100644 --- a/pkg/kubelet/prober/worker_test.go +++ b/pkg/kubelet/prober/worker_test.go @@ -37,25 +37,6 @@ import ( func init() { } -// newTestWorkerWithRestartableInitContainer creates a test worker with an init container setup -func newTestWorkerWithRestartableInitContainer(m *manager, probeType probeType) *worker { - pod := getTestPod() - - // Set up init container with restart policy - initContainer := pod.Spec.Containers[0] - initContainer.Name = testContainerName - restartPolicy := v1.ContainerRestartPolicyAlways - initContainer.RestartPolicy = &restartPolicy - - // Move container to init containers and add a regular container - pod.Spec.InitContainers = []v1.Container{initContainer} - pod.Spec.Containers = []v1.Container{{ - Name: "main-container", - }} - - return newWorker(m, probeType, pod, initContainer) -} - func TestDoProbe(t *testing.T) { logger, ctx := ktesting.NewTestContext(t) m := newTestManager() @@ -549,72 +530,6 @@ func TestResultRunOnStartupCheckFailure(t *testing.T) { } } -func TestDoProbe_TerminatedRestartableInitContainerWithRestartPolicyNever(t *testing.T) { - logger, ctx := ktesting.NewTestContext(t) - m := newTestManager() - - // Test restartable init container (sidecar) behavior - w := newTestWorkerWithRestartableInitContainer(m, startup) - - // Set pod restart policy to Never - w.pod.Spec.RestartPolicy = v1.RestartPolicyNever - - // Create terminated status for init container - terminatedStatus := getTestRunningStatus() - terminatedStatus.InitContainerStatuses = []v1.ContainerStatus{{ - Name: testContainerName, - ContainerID: "test://test_container_id", - State: v1.ContainerState{ - Terminated: &v1.ContainerStateTerminated{ - StartedAt: metav1.Now(), - }, - }, - }} - terminatedStatus.ContainerStatuses[0].State.Running = nil - - m.statusManager.SetPodStatus(logger, w.pod, terminatedStatus) - - // Test: Terminated restartable init container with restartPolicy=Always should continue probing - // even when pod has restartPolicy=Never - if !w.doProbe(ctx) { - t.Error("Expected to continue probing for terminated restartable init container with pod restart policy Never") - } - - // Verify result is set to Failure for terminated container - expectResult(t, w, results.Failure, "restartable init container with pod restart policy Never") -} - -func TestDoProbe_TerminatedContainerWithRestartPolicyNever(t *testing.T) { - logger, ctx := ktesting.NewTestContext(t) - m := newTestManager() - - // Test that regular containers (without RestartPolicy) still work as before - w := newTestWorker(m, startup, v1.Probe{}) - - // Regular container without explicit restart policy - w.container.RestartPolicy = nil - - // Set pod restart policy to Never - w.pod.Spec.RestartPolicy = v1.RestartPolicyNever - - // Create terminated status - terminatedStatus := getTestRunningStatus() - terminatedStatus.ContainerStatuses[0].State.Running = nil - terminatedStatus.ContainerStatuses[0].State.Terminated = &v1.ContainerStateTerminated{ - StartedAt: metav1.Now(), - } - - m.statusManager.SetPodStatus(logger, w.pod, terminatedStatus) - - // Should stop probing (existing behavior preserved) - if w.doProbe(ctx) { - t.Error("Expected to stop probing for regular container with pod RestartPolicy=Never") - } - - // Verify result is set to Failure for terminated container - expectResult(t, w, results.Failure, "regular container with pod restart policy Never") -} - func TestLivenessProbeDisabledByStarted(t *testing.T) { logger, ctx := ktesting.NewTestContext(t) m := newTestManager() diff --git a/pkg/proxy/util/localdetector.go b/pkg/proxy/util/localdetector.go index 9e579e8876123..6e296b0c71b80 100644 --- a/pkg/proxy/util/localdetector.go +++ b/pkg/proxy/util/localdetector.go @@ -98,8 +98,8 @@ func NewDetectLocalByBridgeInterface(interfaceName string) LocalTrafficDetector return &detectLocal{ ifLocal: []string{"-i", interfaceName}, ifNotLocal: []string{"!", "-i", interfaceName}, - ifLocalNFT: []string{"iifname", interfaceName}, - ifNotLocalNFT: []string{"iifname", "!=", interfaceName}, + ifLocalNFT: []string{"iif", interfaceName}, + ifNotLocalNFT: []string{"iif", "!=", interfaceName}, } } @@ -110,7 +110,7 @@ func NewDetectLocalByInterfaceNamePrefix(interfacePrefix string) LocalTrafficDet return &detectLocal{ ifLocal: []string{"-i", interfacePrefix + "+"}, ifNotLocal: []string{"!", "-i", interfacePrefix + "+"}, - ifLocalNFT: []string{"iifname", interfacePrefix + "*"}, - ifNotLocalNFT: []string{"iifname", "!=", interfacePrefix + "*"}, + ifLocalNFT: []string{"iif", interfacePrefix + "*"}, + ifNotLocalNFT: []string{"iif", "!=", interfacePrefix + "*"}, } } diff --git a/pkg/proxy/util/localdetector_test.go b/pkg/proxy/util/localdetector_test.go index 0fb36934262a5..473aaeb8ce45e 100644 --- a/pkg/proxy/util/localdetector_test.go +++ b/pkg/proxy/util/localdetector_test.go @@ -105,37 +105,6 @@ func TestDetectLocalByBridgeInterface(t *testing.T) { } } -func TestDetectLocalNFTByBridgeInterface(t *testing.T) { - cases := []struct { - ifaceName string - expectedJumpIfOutput []string - expectedJumpIfNotOutput []string - }{ - { - ifaceName: "eth0", - expectedJumpIfOutput: []string{"iifname", "eth0"}, - expectedJumpIfNotOutput: []string{"iifname", "!=", "eth0"}, - }, - } - for _, c := range cases { - localDetector := NewDetectLocalByBridgeInterface(c.ifaceName) - if !localDetector.IsImplemented() { - t.Error("DetectLocalByBridgeInterface returns false for IsImplemented") - } - - ifLocal := localDetector.IfLocalNFT() - ifNotLocal := localDetector.IfNotLocalNFT() - - if !reflect.DeepEqual(ifLocal, c.expectedJumpIfOutput) { - t.Errorf("IfLocalNFT, expected: '%v', but got: '%v'", c.expectedJumpIfOutput, ifLocal) - } - - if !reflect.DeepEqual(ifNotLocal, c.expectedJumpIfNotOutput) { - t.Errorf("IfNotLocalNFT, expected: '%v', but got: '%v'", c.expectedJumpIfNotOutput, ifNotLocal) - } - } -} - func TestDetectLocalByInterfaceNamePrefix(t *testing.T) { cases := []struct { ifacePrefix string @@ -168,36 +137,3 @@ func TestDetectLocalByInterfaceNamePrefix(t *testing.T) { } } } - -func TestDetectLocalNFTByInterfaceNamePrefix(t *testing.T) { - cases := []struct { - ifacePrefix string - chain string - args []string - expectedJumpIfOutput []string - expectedJumpIfNotOutput []string - }{ - { - ifacePrefix: "eth", - expectedJumpIfOutput: []string{"iifname", "eth*"}, - expectedJumpIfNotOutput: []string{"iifname", "!=", "eth*"}, - }, - } - for _, c := range cases { - localDetector := NewDetectLocalByInterfaceNamePrefix(c.ifacePrefix) - if !localDetector.IsImplemented() { - t.Error("DetectLocalByInterfaceNamePrefix returns false for IsImplemented") - } - - ifLocal := localDetector.IfLocalNFT() - ifNotLocal := localDetector.IfNotLocalNFT() - - if !reflect.DeepEqual(ifLocal, c.expectedJumpIfOutput) { - t.Errorf("IfLocalNFT, expected: '%v', but got: '%v'", c.expectedJumpIfOutput, ifLocal) - } - - if !reflect.DeepEqual(ifNotLocal, c.expectedJumpIfNotOutput) { - t.Errorf("IfNotLocalNFT, expected: '%v', but got: '%v'", c.expectedJumpIfNotOutput, ifNotLocal) - } - } -} diff --git a/pkg/proxy/winkernel/hns.go b/pkg/proxy/winkernel/hns.go index 80bb208adff10..d6e425f8b77b8 100644 --- a/pkg/proxy/winkernel/hns.go +++ b/pkg/proxy/winkernel/hns.go @@ -349,21 +349,17 @@ func (hns hns) getAllLoadBalancers() (map[loadBalancerIdentifier]*loadBalancerIn func (hns hns) getLoadBalancer(endpoints []endpointInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16, previousLoadBalancers map[loadBalancerIdentifier]*loadBalancerInfo) (*loadBalancerInfo, error) { var id loadBalancerIdentifier vips := []string{} - id, lbIdErr := findLoadBalancerID( - endpoints, - vip, - protocol, - internalPort, - externalPort, - ) - - if lbIdErr != nil { - klog.V(2).ErrorS(lbIdErr, "Error hashing endpoints", "endpoints", endpoints) - return nil, lbIdErr + // Compute hash from backends (endpoint IDs) + hash, err := hashEndpoints(endpoints) + if err != nil { + klog.V(2).ErrorS(err, "Error hashing endpoints", "endpoints", endpoints) + return nil, err } - if len(vip) > 0 { + id = loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, vip: vip, endpointsHash: hash} vips = append(vips, vip) + } else { + id = loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, endpointsHash: hash} } if lb, found := previousLoadBalancers[id]; found { diff --git a/pkg/proxy/winkernel/hns_test.go b/pkg/proxy/winkernel/hns_test.go index b859ad639efb6..ca436394b01a1 100644 --- a/pkg/proxy/winkernel/hns_test.go +++ b/pkg/proxy/winkernel/hns_test.go @@ -39,8 +39,6 @@ const ( epIpv6Address = "192::3" epIpAddressB = "192.168.1.4" epIpAddressRemote = "192.168.2.3" - epIpAddressLocal1 = "192.168.4.4" - epIpAddressLocal2 = "192.168.4.5" epPaAddress = "10.0.0.3" protocol = 6 internalPort = 80 diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index ad3bd26fd61ec..f32a5cb4fc6b4 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -405,7 +405,7 @@ func (proxier *Proxier) updateTerminatedEndpoints(eps []proxy.Endpoint, isOldEnd func (proxier *Proxier) endpointsMapChange(oldEndpointsMap, newEndpointsMap proxy.EndpointsMap) { // This will optimize remote endpoint and loadbalancer deletion based on the annotation var svcPortMap = make(map[proxy.ServicePortName]bool) - + clear(proxier.terminatedEndpoints) var logLevel klog.Level = 5 for svcPortName, eps := range oldEndpointsMap { logFormattedEndpoints("endpointsMapChange oldEndpointsMap", logLevel, svcPortName, eps) @@ -1233,9 +1233,6 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { return } - // Clear terminated endpoints map - clear(proxier.terminatedEndpoints) - // We assume that if this was called, we really want to sync them, // even if nothing changed in the meantime. In other words, callers are // responsible for detecting no-op changes and not calling this function. @@ -1529,7 +1526,7 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { } if !proxier.requiresUpdateLoadbalancer(svcInfo.hnsID, len(clusterIPEndpoints)) { - proxier.deleteExistingLoadBalancer(hns, svcInfo.winProxyOptimization, &svcInfo.hnsID, svcInfo.ClusterIP().String(), Enum(svcInfo.Protocol()), uint16(svcInfo.targetPort), uint16(svcInfo.Port()), clusterIPEndpoints, queriedLoadBalancers) + proxier.deleteExistingLoadBalancer(hns, svcInfo.winProxyOptimization, &svcInfo.hnsID, svcInfo.ClusterIP().String(), Enum(svcInfo.Protocol()), uint16(svcInfo.targetPort), uint16(svcInfo.Port()), hnsEndpoints, queriedLoadBalancers) if len(clusterIPEndpoints) > 0 { // If all endpoints are terminating, then no need to create Cluster IP LoadBalancer @@ -1787,11 +1784,10 @@ func (proxier *Proxier) syncProxyRules() (retryError error) { klog.V(5).InfoS("Terminated endpoints ready for deletion", "epIP", epIP) if epToDelete := queriedEndpoints[epIP]; epToDelete != nil && epToDelete.hnsID != "" && !epToDelete.IsLocal() { if refCount := proxier.endPointsRefCount.getRefCount(epToDelete.hnsID); refCount == nil || *refCount == 0 { + klog.V(3).InfoS("Deleting unreferenced remote endpoint", "hnsID", epToDelete.hnsID, "IP", epToDelete.ip) err := proxier.hns.deleteEndpoint(epToDelete.hnsID) if err != nil { klog.ErrorS(err, "Deleting unreferenced remote endpoint failed", "hnsID", epToDelete.hnsID) - } else { - klog.V(3).InfoS("Deleting unreferenced remote endpoint succeeded", "hnsID", epToDelete.hnsID, "IP", epToDelete.ip) } } } diff --git a/pkg/proxy/winkernel/proxier_test.go b/pkg/proxy/winkernel/proxier_test.go index aaa39ded01caa..d6a2a31e2c34a 100644 --- a/pkg/proxy/winkernel/proxier_test.go +++ b/pkg/proxy/winkernel/proxier_test.go @@ -48,19 +48,15 @@ import ( const ( testNodeName = "test-node" testNetwork = "TestNetwork" + ipAddress = "10.0.0.1" prefixLen = 24 macAddress = "00-11-22-33-44-55" - macAddressLocal1 = "00-11-22-33-44-56" - macAddressLocal2 = "00-11-22-33-44-57" destinationPrefix = "192.168.2.0/24" providerAddress = "10.0.0.3" guid = "123ABC" - networkId = "123ABC" endpointGuid1 = "EPID-1" loadbalancerGuid1 = "LBID-1" - loadbalancerGuid2 = "LBID-2" - endpointLocal1 = "EP-LOCAL-1" - endpointLocal2 = "EP-LOCAL-2" + endpointLocal = "EP-LOCAL" endpointGw = "EP-GW" epIpAddressGw = "192.168.2.1" epMacAddressGw = "00-11-22-33-44-66" @@ -417,193 +413,6 @@ func TestDsrNotAppliedToClusterTrafficPolicy(t *testing.T) { } } -// TestClusterIPSvcWithITPLocal tests the following scenarios for a ClusterIP service with InternalTrafficPolicy=Local: -// 1. When a local endpoint is added to the service, the service should continue to use the local endpoints and existing loadbalancer. -// If no existing loadbalancer is present, a new loadbalancer should be created. -// 2. When one more local endpoint is added to the service, the service should delete existing loadbalancer and create a new loadbalancer. -// 3. When a remote endpoint is added to the service, the service should continue to use the local endpoints and existing loadbalancer, -// since it's a InternalTrafficPolicy=Local service. -func TestClusterIPSvcWithITPLocal(t *testing.T) { - proxier := NewFakeProxier(t, "testhost", netutils.ParseIPSloppy("10.0.0.1"), "L2Bridge", true) - if proxier == nil { - t.Fatal("Failed to create proxier") - } - - svcIP := "10.20.30.41" - svcPort := 80 - svcPortName := proxy.ServicePortName{ - NamespacedName: makeNSN("ns1", "svc1"), - Port: "p80", - Protocol: v1.ProtocolTCP, - } - - itpLocal := v1.ServiceInternalTrafficPolicyLocal - - makeServiceMap(proxier, - makeTestService(svcPortName.Namespace, svcPortName.Name, func(svc *v1.Service) { - svc.Spec.Type = v1.ServiceTypeClusterIP - svc.Spec.ClusterIP = svcIP - svc.Spec.InternalTrafficPolicy = &itpLocal // Setting the InternalTrafficPolicy to Local - svc.Spec.Ports = []v1.ServicePort{{ - Name: svcPortName.Port, - Port: int32(svcPort), - Protocol: v1.ProtocolTCP, - }} - }), - ) - - populateEndpointSlices(proxier, - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{ - { - Addresses: []string{epIpAddressLocal1}, // Local Endpoint 1 - }, - } - eps.Ports = []discovery.EndpointPort{{ - Name: ptr.To(svcPortName.Port), - Port: ptr.To(int32(svcPort)), - Protocol: ptr.To(v1.ProtocolTCP), - }} - }), - ) - - hcn := (proxier.hcn).(*fakehcn.HcnMock) - // Populating the endpoint to the cache, since it's a local endpoint and local endpoints are managed by CNI and not KubeProxy - // Populating here marks the endpoint to local - hcn.PopulateQueriedEndpoints(endpointLocal1, networkId, epIpAddressLocal1, macAddressLocal1, prefixLen) - - proxier.setInitialized(true) - - // Test 1: When a local endpoint is added to the service, the service should continue to use the local endpoints and existing loadbalancer. - // If no existing loadbalancer is present, a new loadbalancer should be created. - proxier.syncProxyRules() - - ep := proxier.endpointsMap[svcPortName][0] - epInfo, ok := ep.(*endpointInfo) - assert.True(t, ok, fmt.Sprintf("Failed to cast endpointInfo %q", svcPortName.String())) - assert.NotEmpty(t, epInfo.hnsID, fmt.Sprintf("Expected HNS ID to be set for endpoint %s, but got empty value", epIpAddressRemote)) - - svc := proxier.svcPortMap[svcPortName] - svcInfo, ok := svc.(*serviceInfo) - assert.True(t, ok, "Failed to cast serviceInfo %q", svcPortName.String()) - assert.Equal(t, svcInfo.hnsID, loadbalancerGuid1, fmt.Sprintf("%v does not match %v", svcInfo.hnsID, loadbalancerGuid1)) - lb, err := proxier.hcn.GetLoadBalancerByID(loadbalancerGuid1) - assert.Equal(t, nil, err, fmt.Sprintf("Failed to fetch loadbalancer: %s. Error: %v", loadbalancerGuid1, err)) - assert.NotNil(t, lb, "Loadbalancer object should not be nil") - - // Test 2: When one more local endpoint is added to the service, the service should delete existing loadbalancer and create a new loadbalancer. - - proxier.setInitialized(false) - - proxier.OnEndpointSliceUpdate( - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{{ - Addresses: []string{epIpAddressLocal1}, - }} - eps.Ports = []discovery.EndpointPort{{ - Name: ptr.To(svcPortName.Port), - Port: ptr.To(int32(svcPort)), - Protocol: ptr.To(v1.ProtocolTCP), - }} - }), - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{ - { - Addresses: []string{epIpAddressLocal1}, - }, - { - Addresses: []string{epIpAddressLocal2}, // Adding one more local endpoint - }, - } - eps.Ports = []discovery.EndpointPort{{ - Name: ptr.To(svcPortName.Port), - Port: ptr.To(int32(svcPort)), - Protocol: ptr.To(v1.ProtocolTCP), - }} - })) - - proxier.mu.Lock() - proxier.endpointSlicesSynced = true - proxier.mu.Unlock() - - proxier.setInitialized(true) - - // Creating the second local endpoint - hcn.PopulateQueriedEndpoints(endpointLocal2, networkId, epIpAddressLocal2, macAddressLocal2, prefixLen) - // Reinitiating the syncProxyRules to create new loadbalancer with the new local endpoint - proxier.syncProxyRules() - svc = proxier.svcPortMap[svcPortName] - svcInfo, ok = svc.(*serviceInfo) - assert.True(t, ok, "Failed to cast serviceInfo %q", svcPortName.String()) - assert.Equal(t, svcInfo.hnsID, loadbalancerGuid2, fmt.Sprintf("%v does not match %v", svcInfo.hnsID, loadbalancerGuid2)) - lb, err = proxier.hcn.GetLoadBalancerByID(loadbalancerGuid2) - assert.Equal(t, nil, err, fmt.Sprintf("Failed to fetch loadbalancer: %s. Error: %v", loadbalancerGuid2, err)) - assert.NotNil(t, lb, "Loadbalancer object should not be nil") - - lb, _ = proxier.hcn.GetLoadBalancerByID(loadbalancerGuid1) - assert.Nil(t, lb, fmt.Sprintf("Loadbalancer object should be nil: %s", loadbalancerGuid1)) - - // Test 3: When a remote endpoint is added to the service, the service should continue to use the local endpoints and existing loadbalancer, - // since it's a InternalTrafficPolicy=Local service. - - proxier.setInitialized(false) - - proxier.OnEndpointSliceUpdate( - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{ - { - Addresses: []string{epIpAddressLocal1}, - }, - { - Addresses: []string{epIpAddressLocal2}, - }, - } - eps.Ports = []discovery.EndpointPort{{ - Name: ptr.To(svcPortName.Port), - Port: ptr.To(int32(svcPort)), - Protocol: ptr.To(v1.ProtocolTCP), - }} - }), - makeTestEndpointSlice(svcPortName.Namespace, svcPortName.Name, 1, func(eps *discovery.EndpointSlice) { - eps.AddressType = discovery.AddressTypeIPv4 - eps.Endpoints = []discovery.Endpoint{ - { - Addresses: []string{epIpAddressLocal1}, - }, - { - Addresses: []string{epIpAddressLocal2}, // Adding one more local endpoint - }, - { - Addresses: []string{epIpAddressRemote}, // Adding one more remote endpoint to the slice - }, - } - eps.Ports = []discovery.EndpointPort{{ - Name: ptr.To(svcPortName.Port), - Port: ptr.To(int32(svcPort)), - Protocol: ptr.To(v1.ProtocolTCP), - }} - })) - - proxier.mu.Lock() - proxier.endpointSlicesSynced = true - proxier.mu.Unlock() - - proxier.setInitialized(true) - - proxier.syncProxyRules() - svc = proxier.svcPortMap[svcPortName] - svcInfo, ok = svc.(*serviceInfo) - assert.True(t, ok, "Failed to cast serviceInfo %q", svcPortName.String()) - assert.Equal(t, svcInfo.hnsID, loadbalancerGuid2, fmt.Sprintf("%v does not match %v", svcInfo.hnsID, loadbalancerGuid2)) - lb, err = proxier.hcn.GetLoadBalancerByID(loadbalancerGuid2) - assert.Equal(t, nil, err, fmt.Sprintf("Failed to fetch loadbalancer: %s. Error: %v", loadbalancerGuid2, err)) - assert.NotNil(t, lb, "Loadbalancer object should not be nil") -} - func TestSharedRemoteEndpointDelete(t *testing.T) { proxier := NewFakeProxier(t, testNodeName, netutils.ParseIPSloppy("10.0.0.1"), "L2Bridge", true) if proxier == nil { @@ -1389,7 +1198,7 @@ func TestCreateDsrLoadBalancer(t *testing.T) { hcn := (proxier.hcn).(*fakehcn.HcnMock) proxier.rootHnsEndpointName = endpointGw - hcn.PopulateQueriedEndpoints(endpointLocal1, guid, epIpAddressRemote, macAddress, prefixLen) + hcn.PopulateQueriedEndpoints(endpointLocal, guid, epIpAddressRemote, macAddress, prefixLen) hcn.PopulateQueriedEndpoints(endpointGw, guid, epIpAddressGw, epMacAddressGw, prefixLen) proxier.setInitialized(true) proxier.syncProxyRules() diff --git a/pkg/registry/batch/job/strategy.go b/pkg/registry/batch/job/strategy.go index 23f343e754d02..8bde82debf726 100644 --- a/pkg/registry/batch/job/strategy.go +++ b/pkg/registry/batch/job/strategy.go @@ -380,11 +380,6 @@ func getStatusValidationOptions(newJob, oldJob *batch.Job) batchvalidation.JobSt isReadyChanged := !ptr.Equal(oldJob.Status.Ready, newJob.Status.Ready) isTerminatingChanged := !ptr.Equal(oldJob.Status.Terminating, newJob.Status.Terminating) isSuspendedWithZeroCompletions := ptr.Equal(newJob.Spec.Suspend, ptr.To(true)) && ptr.Equal(newJob.Spec.Completions, ptr.To[int32](0)) - // Detect job resume via condition changes (JobSuspended: True -> False) - // This handles the case where the controller updates status after the user has already - // changed spec.suspend=false, which is the scenario from https://github.com/kubernetes/kubernetes/issues/134521 - isJobResuming := batchvalidation.IsConditionTrue(oldJob.Status.Conditions, batch.JobSuspended) && - batchvalidation.IsConditionFalse(newJob.Status.Conditions, batch.JobSuspended) return batchvalidation.JobStatusValidationOptions{ // We allow to decrease the counter for succeeded pods for jobs which @@ -402,7 +397,7 @@ func getStatusValidationOptions(newJob, oldJob *batch.Job) batchvalidation.JobSt RejectFinishedJobWithActivePods: isJobFinishedChanged || isActiveChanged, RejectFinishedJobWithoutStartTime: (isJobFinishedChanged || isStartTimeChanged) && !isSuspendedWithZeroCompletions, RejectFinishedJobWithUncountedTerminatedPods: isJobFinishedChanged || isUncountedTerminatedPodsChanged, - RejectStartTimeUpdateForUnsuspendedJob: isStartTimeChanged && !isJobResuming, + RejectStartTimeUpdateForUnsuspendedJob: isStartTimeChanged, RejectCompletionTimeBeforeStartTime: isStartTimeChanged || isCompletionTimeChanged, RejectMutatingCompletionTime: true, RejectNotCompleteJobWithCompletionTime: isJobCompleteChanged || isCompletionTimeChanged, diff --git a/pkg/registry/batch/job/strategy_test.go b/pkg/registry/batch/job/strategy_test.go index 58b4a540448c7..1997106a2da22 100644 --- a/pkg/registry/batch/job/strategy_test.go +++ b/pkg/registry/batch/job/strategy_test.go @@ -2559,33 +2559,6 @@ func TestStatusStrategy_ValidateUpdate(t *testing.T) { {Type: field.ErrorTypeRequired, Field: "status.startTime"}, }, }, - "verify startTime can be updated when resuming job (JobSuspended: True -> False)": { - enableJobManagedBy: true, - job: &batch.Job{ - ObjectMeta: validObjectMeta, - Status: batch.JobStatus{ - StartTime: &now, - Conditions: []batch.JobCondition{ - { - Type: batch.JobSuspended, - Status: api.ConditionTrue, - }, - }, - }, - }, - newJob: &batch.Job{ - ObjectMeta: validObjectMeta, - Status: batch.JobStatus{ - StartTime: &nowPlusMinute, - Conditions: []batch.JobCondition{ - { - Type: batch.JobSuspended, - Status: api.ConditionFalse, - }, - }, - }, - }, - }, "invalid attempt to set completionTime before startTime": { enableJobManagedBy: true, job: &batch.Job{ diff --git a/pkg/scheduler/backend/api_dispatcher/api_dispatcher_test.go b/pkg/scheduler/backend/api_dispatcher/api_dispatcher_test.go index d04544f1a74db..0134ec4c4eecd 100644 --- a/pkg/scheduler/backend/api_dispatcher/api_dispatcher_test.go +++ b/pkg/scheduler/backend/api_dispatcher/api_dispatcher_test.go @@ -22,21 +22,19 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - utilfeature "k8s.io/apiserver/pkg/util/feature" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" - featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/component-base/metrics/testutil" "k8s.io/klog/v2/ktesting" fwk "k8s.io/kube-scheduler/framework" - "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler/metrics" ) -func registerAndResetMetrics(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncAPICalls, true) +func init() { metrics.Register() +} +func resetMetrics() { metrics.AsyncAPICallsTotal.Reset() metrics.AsyncAPICallDuration.Reset() metrics.AsyncAPIPendingCalls.Reset() @@ -102,7 +100,7 @@ func (mac *mockAPICall) IsNoOp() bool { func TestAPIDispatcherLifecycle(t *testing.T) { // Reset all async API metrics - registerAndResetMetrics(t) + resetMetrics() logger, _ := ktesting.NewTestContext(t) diff --git a/pkg/scheduler/backend/api_dispatcher/call_queue_test.go b/pkg/scheduler/backend/api_dispatcher/call_queue_test.go index 915ee7970826f..926f3cff7558a 100644 --- a/pkg/scheduler/backend/api_dispatcher/call_queue_test.go +++ b/pkg/scheduler/backend/api_dispatcher/call_queue_test.go @@ -106,7 +106,7 @@ func TestCallQueueAdd(t *testing.T) { uid2 := types.UID("uid2") t.Run("First call is added without collision", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) call := &queuedAPICall{ @@ -124,7 +124,7 @@ func TestCallQueueAdd(t *testing.T) { }) t.Run("No-op call is skipped", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) onFinishCh := make(chan error, 1) @@ -149,7 +149,7 @@ func TestCallQueueAdd(t *testing.T) { }) t.Run("Two calls for different objects don't collide", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) call1 := &queuedAPICall{ @@ -176,7 +176,7 @@ func TestCallQueueAdd(t *testing.T) { }) t.Run("New call overwrites less relevant call", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) onFinishCh := make(chan error, 1) @@ -206,7 +206,7 @@ func TestCallQueueAdd(t *testing.T) { }) t.Run("New call is skipped if less relevant", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) onFinishCh := make(chan error, 1) @@ -237,7 +237,7 @@ func TestCallQueueAdd(t *testing.T) { }) t.Run("New call merges with old call and skips if no-op", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) onFinishCh1 := make(chan error, 1) @@ -308,7 +308,7 @@ func TestCallQueuePop(t *testing.T) { uid2 := types.UID("uid2") t.Run("Calls are popped from the queue in FIFO order", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) call1 := &queuedAPICall{ @@ -395,7 +395,7 @@ func TestCallQueueFinalize(t *testing.T) { uid := types.UID("uid") t.Run("Call details are cleared if there is no waiting call", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) call := &queuedAPICall{ @@ -420,7 +420,7 @@ func TestCallQueueFinalize(t *testing.T) { }) t.Run("UID is re-queued if a new call arrived while one was in-flight", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) call1 := &queuedAPICall{ @@ -462,7 +462,7 @@ func TestCallQueueSyncObject(t *testing.T) { uid2 := types.UID("uid2") t.Run("Object is synced with pending call details", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) obj := &metav1.ObjectMeta{ @@ -497,7 +497,7 @@ func TestCallQueueSyncObject(t *testing.T) { }) t.Run("Pending call is canceled if sync results in no-op", func(t *testing.T) { - registerAndResetMetrics(t) + resetMetrics() cq := newCallQueue(mockRelevances) obj := &metav1.ObjectMeta{UID: uid1} diff --git a/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration.go b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration.go index 48b9f2fc86c2d..9e75df58db2a2 100644 --- a/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration.go +++ b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration.go @@ -116,8 +116,8 @@ func (pl *TaintToleration) Filter(ctx context.Context, state fwk.CycleState, pod return nil } - klog.FromContext(ctx).V(4).Info("node had untolerated taints", "node", klog.KObj(node), "pod", klog.KObj(pod), "untoleratedTaint", taint) - return fwk.NewStatus(fwk.UnschedulableAndUnresolvable, "node(s) had untolerated taint(s)") + errReason := fmt.Sprintf("node(s) had untolerated taint {%s: %s}", taint.Key, taint.Value) + return fwk.NewStatus(fwk.UnschedulableAndUnresolvable, errReason) } // preScoreState computed at PreScore and used at Score. diff --git a/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go index 3af361cc7b1c5..058a0ef17b822 100644 --- a/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go +++ b/pkg/scheduler/framework/plugins/tainttoleration/taint_toleration_test.go @@ -283,7 +283,7 @@ func TestTaintTolerationFilter(t *testing.T) { pod: podWithTolerations("pod1", []v1.Toleration{}), node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}), wantStatus: fwk.NewStatus(fwk.UnschedulableAndUnresolvable, - "node(s) had untolerated taint(s)"), + "node(s) had untolerated taint {dedicated: user1}"), }, { name: "A pod which can be scheduled on a dedicated node assigned to user1 with effect NoSchedule", @@ -295,7 +295,7 @@ func TestTaintTolerationFilter(t *testing.T) { pod: podWithTolerations("pod1", []v1.Toleration{{Key: "dedicated", Operator: "Equal", Value: "user2", Effect: "NoSchedule"}}), node: nodeWithTaints("nodeA", []v1.Taint{{Key: "dedicated", Value: "user1", Effect: "NoSchedule"}}), wantStatus: fwk.NewStatus(fwk.UnschedulableAndUnresolvable, - "node(s) had untolerated taint(s)"), + "node(s) had untolerated taint {dedicated: user1}"), }, { name: "A pod can be scheduled onto the node, with a toleration uses operator Exists that tolerates the taints on the node", @@ -319,7 +319,7 @@ func TestTaintTolerationFilter(t *testing.T) { pod: podWithTolerations("pod1", []v1.Toleration{{Key: "foo", Operator: "Equal", Value: "bar", Effect: "PreferNoSchedule"}}), node: nodeWithTaints("nodeA", []v1.Taint{{Key: "foo", Value: "bar", Effect: "NoSchedule"}}), wantStatus: fwk.NewStatus(fwk.UnschedulableAndUnresolvable, - "node(s) had untolerated taint(s)"), + "node(s) had untolerated taint {foo: bar}"), }, { name: "The pod has a toleration that keys and values match the taint on the node, the effect of toleration is empty, " + diff --git a/pkg/scheduler/framework/preemption/preemption.go b/pkg/scheduler/framework/preemption/preemption.go index ee98910ef993a..9bcd91592fec1 100644 --- a/pkg/scheduler/framework/preemption/preemption.go +++ b/pkg/scheduler/framework/preemption/preemption.go @@ -189,12 +189,12 @@ func NewEvaluator(pluginName string, fh framework.Handle, i Interface, enableAsy } } if err := util.DeletePod(ctx, ev.Handler.ClientSet(), victim); err != nil { - if !apierrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { + logger.V(2).Info("Victim Pod is already deleted", "preemptor", klog.KObj(preemptor), "victim", klog.KObj(victim), "node", c.Name()) + } else { logger.Error(err, "Tried to preempted pod", "pod", klog.KObj(victim), "preemptor", klog.KObj(preemptor)) - return err } - logger.V(2).Info("Victim Pod is already deleted", "preemptor", klog.KObj(preemptor), "victim", klog.KObj(victim), "node", c.Name()) - return nil + return err } logger.V(2).Info("Preemptor Pod preempted victim Pod", "preemptor", klog.KObj(preemptor), "victim", klog.KObj(victim), "node", c.Name()) } @@ -436,7 +436,14 @@ func (ev *Evaluator) prepareCandidate(ctx context.Context, c Candidate, pod *v1. logger := klog.FromContext(ctx) errCh := parallelize.NewErrorChannel() fh.Parallelizer().Until(ctx, len(c.Victims().Pods), func(index int) { - if err := ev.PreemptPod(ctx, c, pod, c.Victims().Pods[index], pluginName); err != nil { + victimPod := c.Victims().Pods[index] + if victimPod.DeletionTimestamp != nil { + // If the victim Pod is already being deleted, we don't have to make another deletion api call. + logger.V(2).Info("Victim Pod is already deleted, skipping the API call for it", "preemptor", klog.KObj(pod), "node", c.Name(), "victim", klog.KObj(victimPod)) + return + } + + if err := ev.PreemptPod(ctx, c, pod, victimPod, pluginName); err != nil && !apierrors.IsNotFound(err) { errCh.SendErrorWithCancel(err, cancel) } }, ev.PluginName) @@ -497,11 +504,34 @@ func (ev *Evaluator) prepareCandidateAsync(c Candidate, pod *v1.Pod, pluginName // Intentionally create a new context, not using a ctx from the scheduling cycle, to create ctx, // because this process could continue even after this scheduling cycle finishes. ctx, cancel := context.WithCancel(context.Background()) + logger := klog.FromContext(ctx) + victimPods := make([]*v1.Pod, 0, len(c.Victims().Pods)) + for _, victim := range c.Victims().Pods { + if victim.DeletionTimestamp != nil { + // If the victim Pod is already being deleted, we don't have to make another deletion api call. + logger.V(2).Info("Victim Pod is already deleted, skipping the API call for it", "preemptor", klog.KObj(pod), "node", c.Name(), "victim", klog.KObj(victim)) + continue + } + victimPods = append(victimPods, victim) + } + if len(victimPods) == 0 { + cancel() + return + } + errCh := parallelize.NewErrorChannel() + // Whether all victim pods are already deleted before making API call. + var allPodsAlreadyDeleted atomic.Bool + allPodsAlreadyDeleted.Store(true) preemptPod := func(index int) { - victim := c.Victims().Pods[index] - if err := ev.PreemptPod(ctx, c, pod, victim, pluginName); err != nil { + victim := victimPods[index] + err := ev.PreemptPod(ctx, c, pod, victim, pluginName) + switch { + case err != nil && !apierrors.IsNotFound(err): + // We don't have to handle NotFound error here, because it means the victim Pod is already deleted, and the preemption didn't have to remove it. errCh.SendErrorWithCancel(err, cancel) + case err == nil: + allPodsAlreadyDeleted.Store(false) } } @@ -509,21 +539,24 @@ func (ev *Evaluator) prepareCandidateAsync(c Candidate, pod *v1.Pod, pluginName ev.preempting.Insert(pod.UID) ev.mu.Unlock() - logger := klog.FromContext(ctx) go func() { startTime := time.Now() result := metrics.GoroutineResultSuccess + defer metrics.PreemptionGoroutinesDuration.WithLabelValues(result).Observe(metrics.SinceInSeconds(startTime)) defer metrics.PreemptionGoroutinesExecutionTotal.WithLabelValues(result).Inc() defer func() { - if result == metrics.GoroutineResultError { - // When API call isn't successful, the Pod may get stuck in the unschedulable pod pool in the worst case. - // So, we should move the Pod to the activeQ. + // When API call isn't successful, the Pod may get stuck in the unschedulable pod pool in the worst case. + // So, we should move the Pod to the activeQ. + if result == metrics.GoroutineResultError || + // When all pods are already deleted (which is very rare, but could happen in theory), + // it's safe to activate the preemptor Pod because it might miss Pod/delete event that requeues the pod. + allPodsAlreadyDeleted.Load() { ev.Handler.Activate(logger, map[string]*v1.Pod{pod.Name: pod}) } }() defer cancel() - logger.V(2).Info("Start the preemption asynchronously", "preemptor", klog.KObj(pod), "node", c.Name(), "numVictims", len(c.Victims().Pods)) + logger.V(2).Info("Start the preemption asynchronously", "preemptor", klog.KObj(pod), "node", c.Name(), "numVictims", len(c.Victims().Pods), "numVictimsToDelete", len(victimPods)) // Lower priority pods nominated to run on this node, may no longer fit on // this node. So, we should remove their nomination. Removing their @@ -536,33 +569,32 @@ func (ev *Evaluator) prepareCandidateAsync(c Candidate, pod *v1.Pod, pluginName // We do not return as this error is not critical. } - if len(c.Victims().Pods) == 0 { - ev.mu.Lock() - delete(ev.preempting, pod.UID) - ev.mu.Unlock() - - return - } - - // We can evict all victims in parallel, but the last one. - // We have to remove the pod from the preempting map before the last one is evicted - // because, otherwise, the pod removal might be notified to the scheduling queue before - // we remove this pod from the preempting map, - // and the pod could end up stucking at the unschedulable pod pool - // by all the pod removal events being ignored. - ev.Handler.Parallelizer().Until(ctx, len(c.Victims().Pods)-1, preemptPod, ev.PluginName) - if err := errCh.ReceiveError(); err != nil { - utilruntime.HandleErrorWithContext(ctx, err, "Error occurred during async preemption") - result = metrics.GoroutineResultError + if len(victimPods) > 1 { + // We can evict all victims in parallel, but the last one. + // We have to remove the pod from the preempting map before the last one is evicted + // because, otherwise, the pod removal might be notified to the scheduling queue before + // we remove this pod from the preempting map, + // and the pod could end up stucking at the unschedulable pod pool + // by all the pod removal events being ignored. + ev.Handler.Parallelizer().Until(ctx, len(victimPods)-1, preemptPod, ev.PluginName) + if err := errCh.ReceiveError(); err != nil { + utilruntime.HandleErrorWithContext(ctx, err, "Error occurred during async preemption") + result = metrics.GoroutineResultError + } } ev.mu.Lock() delete(ev.preempting, pod.UID) ev.mu.Unlock() - if err := ev.PreemptPod(ctx, c, pod, c.Victims().Pods[len(c.Victims().Pods)-1], pluginName); err != nil { + err := ev.PreemptPod(ctx, c, pod, victimPods[len(victimPods)-1], pluginName) + switch { + case err != nil && !apierrors.IsNotFound(err): + // We don't have to handle NotFound error here, because it means the victim Pod is already deleted, and the preemption didn't have to remove it. utilruntime.HandleErrorWithContext(ctx, err, "Error occurred during async preemption") result = metrics.GoroutineResultError + case err == nil: + allPodsAlreadyDeleted.Store(false) } logger.V(2).Info("Async Preemption finished completely", "preemptor", klog.KObj(pod), "node", c.Name(), "result", result) diff --git a/pkg/scheduler/framework/preemption/preemption_test.go b/pkg/scheduler/framework/preemption/preemption_test.go index e8b0a80ecad20..5a37f0df4f348 100644 --- a/pkg/scheduler/framework/preemption/preemption_test.go +++ b/pkg/scheduler/framework/preemption/preemption_test.go @@ -30,6 +30,7 @@ import ( v1 "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -44,12 +45,12 @@ import ( "k8s.io/klog/v2/ktesting" extenderv1 "k8s.io/kube-scheduler/extender/v1" fwk "k8s.io/kube-scheduler/framework" - "k8s.io/kubernetes/pkg/scheduler/backend/api_cache" - "k8s.io/kubernetes/pkg/scheduler/backend/api_dispatcher" + apicache "k8s.io/kubernetes/pkg/scheduler/backend/api_cache" + apidispatcher "k8s.io/kubernetes/pkg/scheduler/backend/api_dispatcher" internalcache "k8s.io/kubernetes/pkg/scheduler/backend/cache" internalqueue "k8s.io/kubernetes/pkg/scheduler/backend/queue" "k8s.io/kubernetes/pkg/scheduler/framework" - "k8s.io/kubernetes/pkg/scheduler/framework/api_calls" + apicalls "k8s.io/kubernetes/pkg/scheduler/framework/api_calls" "k8s.io/kubernetes/pkg/scheduler/framework/parallelize" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" @@ -420,6 +421,10 @@ func TestPrepareCandidate(t *testing.T) { Containers([]v1.Container{st.MakeContainer().Name("container1").Obj()}). Obj() + notFoundVictim1 = st.MakePod().Name("not-found-victim").UID("victim1"). + Node(node1Name).SchedulerName(defaultSchedulerName).Priority(midPriority). + Containers([]v1.Container{st.MakeContainer().Name("container1").Obj()}). + Obj() failVictim = st.MakePod().Name("fail-victim").UID("victim1"). Node(node1Name).SchedulerName(defaultSchedulerName).Priority(midPriority). Containers([]v1.Container{st.MakeContainer().Name("container1").Obj()}). @@ -451,6 +456,12 @@ func TestPrepareCandidate(t *testing.T) { errPatchStatusFailed = errors.New("patch pod status failed") ) + victimWithDeletionTimestamp := victim1.DeepCopy() + victimWithDeletionTimestamp.Name = "victim1-with-deletion-timestamp" + victimWithDeletionTimestamp.UID = "victim1-with-deletion-timestamp" + victimWithDeletionTimestamp.DeletionTimestamp = &metav1.Time{Time: time.Now().Add(-100 * time.Second)} + victimWithDeletionTimestamp.Finalizers = []string{"test"} + tests := []struct { name string nodeNames []string @@ -479,9 +490,8 @@ func TestPrepareCandidate(t *testing.T) { testPods: []*v1.Pod{ victim1, }, - nodeNames: []string{node1Name}, - expectedStatus: nil, - expectedPreemptingMap: sets.New(types.UID("preemptor")), + nodeNames: []string{node1Name}, + expectedStatus: nil, }, { name: "one victim without condition", @@ -503,6 +513,42 @@ func TestPrepareCandidate(t *testing.T) { expectedStatus: nil, expectedPreemptingMap: sets.New(types.UID("preemptor")), }, + { + name: "one victim, but victim is already being deleted", + + candidate: &fakeCandidate{ + name: node1Name, + victims: &extenderv1.Victims{ + Pods: []*v1.Pod{ + victimWithDeletionTimestamp, + }, + }, + }, + preemptor: preemptor, + testPods: []*v1.Pod{ + victimWithDeletionTimestamp, + }, + nodeNames: []string{node1Name}, + expectedStatus: nil, + }, + { + name: "one victim, but victim is already deleted", + + candidate: &fakeCandidate{ + name: node1Name, + victims: &extenderv1.Victims{ + Pods: []*v1.Pod{ + notFoundVictim1, + }, + }, + }, + preemptor: preemptor, + testPods: []*v1.Pod{}, + nodeNames: []string{node1Name}, + expectedStatus: nil, + expectedActivatedPods: map[string]*v1.Pod{preemptor.Name: preemptor}, + expectedPreemptingMap: sets.New(types.UID("preemptor")), + }, { name: "one victim with same condition", @@ -663,6 +709,11 @@ func TestPrepareCandidate(t *testing.T) { deletionFailure = true return true, nil, errDeletePodFailed } + // fake clientset does not return an error for not-found pods, so we simulate it here. + if name == "not-found-victim" { + // Simulate a not-found error. + return true, nil, apierrors.NewNotFound(v1.Resource("pods"), name) + } deletedPods.Insert(name) return true, nil, nil @@ -675,6 +726,10 @@ func TestPrepareCandidate(t *testing.T) { patchFailure = true return true, nil, errPatchStatusFailed } + // fake clientset does not return an error for not-found pods, so we simulate it here. + if action.(clienttesting.PatchAction).GetName() == "not-found-victim" { + return true, nil, apierrors.NewNotFound(v1.Resource("pods"), "not-found-victim") + } return true, nil, nil }) diff --git a/pkg/volume/csi/csi_attacher.go b/pkg/volume/csi/csi_attacher.go index c4fd435064f1c..314fe68a85e99 100644 --- a/pkg/volume/csi/csi_attacher.go +++ b/pkg/volume/csi/csi_attacher.go @@ -252,7 +252,7 @@ func (c *csiAttacher) VolumesAreAttached(specs []*volume.Spec, nodeName types.No } func (c *csiAttacher) GetDeviceMountPath(spec *volume.Spec) (string, error) { - klog.V(4).Info(log("attacher.GetDeviceMountPath for volume(%s)", spec.Name())) + klog.V(4).Info(log("attacher.GetDeviceMountPath(%v)", spec)) deviceMountPath, err := makeDeviceMountPath(c.plugin, spec) if err != nil { return "", errors.New(log("attacher.GetDeviceMountPath failed to make device mount path: %v", err)) diff --git a/pkg/volume/plugins.go b/pkg/volume/plugins.go index 24b661493fdaa..a6426cbd9d0ed 100644 --- a/pkg/volume/plugins.go +++ b/pkg/volume/plugins.go @@ -997,7 +997,7 @@ func NewPersistentVolumeRecyclerPodTemplate() *v1.Pod { Containers: []v1.Container{ { Name: "pv-recycler", - Image: "registry.k8s.io/build-image/debian-base:bookworm-v1.0.6", + Image: "registry.k8s.io/build-image/debian-base:bookworm-v1.0.4", Command: []string{"/bin/sh"}, Args: []string{"-c", "test -e /scrub && find /scrub -mindepth 1 -delete && test -z \"$(ls -A /scrub)\" || exit 1"}, VolumeMounts: []v1.VolumeMount{ diff --git a/pkg/volume/portworx/portworx.go b/pkg/volume/portworx/portworx.go index f8d4c348c4f56..c41f06b3a0e0e 100644 --- a/pkg/volume/portworx/portworx.go +++ b/pkg/volume/portworx/portworx.go @@ -308,9 +308,8 @@ func (b *portworxVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterAr notMnt, err := b.mounter.IsLikelyNotMountPoint(dir) klog.Infof("Portworx Volume set up. Dir: %s %v %v", dir, !notMnt, err) if err != nil && !os.IsNotExist(err) { - // don't log error details from client calls in events - klog.V(4).Infof("Cannot validate mountpoint %s: %v", dir, err) - return fmt.Errorf("failed to validate mountpoint: see kube-controller-manager.log for details") + klog.Errorf("Cannot validate mountpoint: %s", dir) + return err } if !notMnt { return nil @@ -320,9 +319,7 @@ func (b *portworxVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterAr attachOptions[attachContextKey] = dir attachOptions[attachHostKey] = b.plugin.host.GetHostName() if _, err := b.manager.AttachVolume(b, attachOptions); err != nil { - // don't log error details from client calls in events - klog.V(4).Infof("Failed to attach volume %s: %v", b.volumeID, err) - return fmt.Errorf("failed to attach volume: see kube-controller-manager.log for details") + return err } klog.V(4).Infof("Portworx Volume %s attached", b.volumeID) @@ -332,9 +329,7 @@ func (b *portworxVolumeMounter) SetUpAt(dir string, mounterArgs volume.MounterAr } if err := b.manager.MountVolume(b, dir); err != nil { - // don't log error details from client calls in events - klog.V(4).Infof("Failed to mount volume %s: %v", b.volumeID, err) - return fmt.Errorf("failed to mount volume: see kube-controller-manager.log for details") + return err } if !b.readOnly { // Since portworxVolume is in process of being removed from in-tree, we avoid larger refactor to add progress tracking for ownership operation @@ -367,16 +362,12 @@ func (c *portworxVolumeUnmounter) TearDownAt(dir string) error { klog.Infof("Portworx Volume TearDown of %s", dir) if err := c.manager.UnmountVolume(c, dir); err != nil { - // don't log error details from client calls in events - klog.V(4).Infof("Failed to unmount volume %s: %v", c.volumeID, err) - return fmt.Errorf("failed to unmount volume: see kube-controller-manager.log for details") + return err } // Call Portworx Detach Volume. if err := c.manager.DetachVolume(c); err != nil { - // don't log error details from client calls in events - klog.V(4).Infof("Failed to detach volume %s: %v", c.volumeID, err) - return fmt.Errorf("failed to detach volume: see kube-controller-manager.log for details") + return err } return nil @@ -393,13 +384,7 @@ func (d *portworxVolumeDeleter) GetPath() string { } func (d *portworxVolumeDeleter) Delete() error { - err := d.manager.DeleteVolume(d) - if err != nil { - // don't log error details from client calls in events - klog.V(4).Infof("Failed to delete volume %s: %v", d.volumeID, err) - return fmt.Errorf("failed to delete volume: see kube-controller-manager.log for details") - } - return nil + return d.manager.DeleteVolume(d) } type portworxVolumeProvisioner struct { @@ -420,9 +405,7 @@ func (c *portworxVolumeProvisioner) Provision(selectedNode *v1.Node, allowedTopo volumeID, sizeGiB, labels, err := c.manager.CreateVolume(c) if err != nil { - // don't log error details from client calls in events - klog.V(4).Infof("Failed to create volume: %v", err) - return nil, fmt.Errorf("failed to create volume: see kube-controller-manager.log for details") + return nil, err } pv := &v1.PersistentVolume{ diff --git a/staging/publishing/rules.yaml b/staging/publishing/rules.yaml index 76723a7c5d3a2..f9952f7fb5fbe 100644 --- a/staging/publishing/rules.yaml +++ b/staging/publishing/rules.yaml @@ -19,13 +19,13 @@ rules: dirs: - staging/src/k8s.io/apimachinery - name: release-1.33 - go: 1.24.9 + go: 1.24.6 source: branch: release-1.33 dirs: - staging/src/k8s.io/apimachinery - name: release-1.34 - go: 1.24.9 + go: 1.24.6 source: branch: release-1.34 dirs: @@ -60,7 +60,7 @@ rules: dirs: - staging/src/k8s.io/api - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -69,7 +69,7 @@ rules: dirs: - staging/src/k8s.io/api - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -125,7 +125,7 @@ rules: go build -mod=mod ./... go test -mod=mod ./... - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -140,7 +140,7 @@ rules: go build -mod=mod ./... go test -mod=mod ./... - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -184,7 +184,7 @@ rules: dirs: - staging/src/k8s.io/code-generator - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -193,7 +193,7 @@ rules: dirs: - staging/src/k8s.io/code-generator - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -242,7 +242,7 @@ rules: dirs: - staging/src/k8s.io/component-base - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -255,7 +255,7 @@ rules: dirs: - staging/src/k8s.io/component-base - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -309,7 +309,7 @@ rules: dirs: - staging/src/k8s.io/component-helpers - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -322,7 +322,7 @@ rules: dirs: - staging/src/k8s.io/component-helpers - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -364,7 +364,7 @@ rules: dirs: - staging/src/k8s.io/kms - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -373,7 +373,7 @@ rules: dirs: - staging/src/k8s.io/kms - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -435,7 +435,7 @@ rules: dirs: - staging/src/k8s.io/apiserver - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -452,7 +452,7 @@ rules: dirs: - staging/src/k8s.io/apiserver - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -534,7 +534,7 @@ rules: dirs: - staging/src/k8s.io/kube-aggregator - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -555,7 +555,7 @@ rules: dirs: - staging/src/k8s.io/kube-aggregator - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -655,7 +655,7 @@ rules: # assumes GO111MODULE=on go build -mod=mod . - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -681,7 +681,7 @@ rules: # assumes GO111MODULE=on go build -mod=mod . - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -768,7 +768,7 @@ rules: # assumes GO111MODULE=on go build -mod=mod . - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -788,7 +788,7 @@ rules: # assumes GO111MODULE=on go build -mod=mod . - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -878,7 +878,7 @@ rules: required-packages: - k8s.io/code-generator - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -901,7 +901,7 @@ rules: required-packages: - k8s.io/code-generator - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -970,7 +970,7 @@ rules: dirs: - staging/src/k8s.io/metrics - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -985,7 +985,7 @@ rules: dirs: - staging/src/k8s.io/metrics - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -1041,7 +1041,7 @@ rules: dirs: - staging/src/k8s.io/cli-runtime - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.33 @@ -1054,7 +1054,7 @@ rules: dirs: - staging/src/k8s.io/cli-runtime - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.34 @@ -1114,7 +1114,7 @@ rules: dirs: - staging/src/k8s.io/sample-cli-plugin - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.33 @@ -1129,7 +1129,7 @@ rules: dirs: - staging/src/k8s.io/sample-cli-plugin - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.34 @@ -1190,7 +1190,7 @@ rules: dirs: - staging/src/k8s.io/kube-proxy - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -1205,7 +1205,7 @@ rules: dirs: - staging/src/k8s.io/kube-proxy - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -1240,13 +1240,13 @@ rules: dirs: - staging/src/k8s.io/cri-api - name: release-1.33 - go: 1.24.9 + go: 1.24.6 source: branch: release-1.33 dirs: - staging/src/k8s.io/cri-api - name: release-1.34 - go: 1.24.9 + go: 1.24.6 source: branch: release-1.34 dirs: @@ -1305,7 +1305,7 @@ rules: dirs: - staging/src/k8s.io/cri-client - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.33 @@ -1322,7 +1322,7 @@ rules: dirs: - staging/src/k8s.io/cri-client - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.34 @@ -1404,7 +1404,7 @@ rules: dirs: - staging/src/k8s.io/kubelet - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -1425,7 +1425,7 @@ rules: dirs: - staging/src/k8s.io/kubelet - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -1493,7 +1493,7 @@ rules: dirs: - staging/src/k8s.io/kube-scheduler - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -1508,7 +1508,7 @@ rules: dirs: - staging/src/k8s.io/kube-scheduler - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -1582,7 +1582,7 @@ rules: dirs: - staging/src/k8s.io/controller-manager - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.33 @@ -1601,7 +1601,7 @@ rules: dirs: - staging/src/k8s.io/controller-manager - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.34 @@ -1691,7 +1691,7 @@ rules: dirs: - staging/src/k8s.io/cloud-provider - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.33 @@ -1714,7 +1714,7 @@ rules: dirs: - staging/src/k8s.io/cloud-provider - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.34 @@ -1814,7 +1814,7 @@ rules: dirs: - staging/src/k8s.io/kube-controller-manager - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -1839,7 +1839,7 @@ rules: dirs: - staging/src/k8s.io/kube-controller-manager - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -1899,7 +1899,7 @@ rules: dirs: - staging/src/k8s.io/cluster-bootstrap - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -1910,7 +1910,7 @@ rules: dirs: - staging/src/k8s.io/cluster-bootstrap - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -1956,7 +1956,7 @@ rules: dirs: - staging/src/k8s.io/csi-translation-lib - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.33 @@ -1967,7 +1967,7 @@ rules: dirs: - staging/src/k8s.io/csi-translation-lib - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.34 @@ -1998,13 +1998,13 @@ rules: dirs: - staging/src/k8s.io/mount-utils - name: release-1.33 - go: 1.24.9 + go: 1.24.6 source: branch: release-1.33 dirs: - staging/src/k8s.io/mount-utils - name: release-1.34 - go: 1.24.9 + go: 1.24.6 source: branch: release-1.34 dirs: @@ -2081,7 +2081,7 @@ rules: dirs: - staging/src/k8s.io/kubectl - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.33 @@ -2104,7 +2104,7 @@ rules: dirs: - staging/src/k8s.io/kubectl - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.34 @@ -2186,7 +2186,7 @@ rules: dirs: - staging/src/k8s.io/pod-security-admission - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.33 @@ -2205,7 +2205,7 @@ rules: dirs: - staging/src/k8s.io/pod-security-admission - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.34 @@ -2301,7 +2301,7 @@ rules: dirs: - staging/src/k8s.io/dynamic-resource-allocation - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.33 @@ -2326,7 +2326,7 @@ rules: dirs: - staging/src/k8s.io/dynamic-resource-allocation - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: apimachinery branch: release-1.34 @@ -2397,7 +2397,7 @@ rules: dirs: - staging/src/k8s.io/endpointslice - name: release-1.33 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.33 @@ -2412,7 +2412,7 @@ rules: dirs: - staging/src/k8s.io/endpointslice - name: release-1.34 - go: 1.24.9 + go: 1.24.6 dependencies: - repository: api branch: release-1.34 @@ -2440,17 +2440,17 @@ rules: dirs: - staging/src/k8s.io/externaljwt - name: release-1.33 - go: 1.24.9 + go: 1.24.6 source: branch: release-1.33 dirs: - staging/src/k8s.io/externaljwt - name: release-1.34 - go: 1.24.9 + go: 1.24.6 source: branch: release-1.34 dirs: - staging/src/k8s.io/externaljwt recursive-delete-patterns: - '*/.gitattributes' -default-go-version: 1.24.9 +default-go-version: 1.24.6 diff --git a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics_test.go b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics_test.go index b8f7e72315ecc..9fa48014dc291 100644 --- a/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics_test.go +++ b/staging/src/k8s.io/apiserver/plugin/pkg/authorizer/webhook/metrics_test.go @@ -107,16 +107,9 @@ func TestAuthorizerMetrics(t *testing.T) { if service.statusCode == 0 { service.statusCode = 200 } - - testFinishedCtx, testFinishedCancel := context.WithCancel(context.Background()) - defer testFinishedCancel() - if scenario.canceledRequest { - service.reviewHook = func(*authorizationv1.SubjectAccessReview) { + service.reviewHook = func(*authorizationv1.SubjectAccessReview) { + if scenario.canceledRequest { cancel() - // net/http transport still attempts to use a response if it's - // available right when it's handling context cancellation, - // we need to delay the response. - <-testFinishedCtx.Done() } } service.allow = !scenario.authFakeServiceDeny @@ -127,9 +120,6 @@ func TestAuthorizerMetrics(t *testing.T) { return } defer server.Close() - // testFinishedCtx must be cancelled before we close the server, otherwise - // closing the server hangs on an active connection from the listener. - defer testFinishedCancel() fakeAuthzMetrics := &fakeAuthorizerMetrics{} wh, err := newV1Authorizer(server.URL, scenario.clientCert, scenario.clientKey, scenario.clientCA, 0, fakeAuthzMetrics, cel.NewDefaultCompiler(), []apiserver.WebhookMatchCondition{}, "") diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/resourcelock/leaselock.go b/staging/src/k8s.io/client-go/tools/leaderelection/resourcelock/leaselock.go index 79a748b74d8c1..5d2054155c459 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/resourcelock/leaselock.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/resourcelock/leaselock.go @@ -77,9 +77,6 @@ func (ll *LeaseLock) Update(ctx context.Context, ler LeaderElectionRecord) error ll.lease.Spec = LeaderElectionRecordToLeaseSpec(&ler) if ll.Labels != nil { - if ll.lease.Labels == nil { - ll.lease.Labels = map[string]string{} - } // Only overwrite the labels that are specifically set for k, v := range ll.Labels { ll.lease.Labels[k] = v diff --git a/staging/src/k8s.io/client-go/tools/leaderelection/resourcelock/leaselock_test.go b/staging/src/k8s.io/client-go/tools/leaderelection/resourcelock/leaselock_test.go index f30bcfe284171..b6d1af532e175 100644 --- a/staging/src/k8s.io/client-go/tools/leaderelection/resourcelock/leaselock_test.go +++ b/staging/src/k8s.io/client-go/tools/leaderelection/resourcelock/leaselock_test.go @@ -266,7 +266,7 @@ func TestLeaseConversion(t *testing.T) { } } -func TestUpdateWithNilLabelsOnLease(t *testing.T) { +func TestUpdateWithNilLabels(t *testing.T) { setup() // Create initial lease @@ -278,33 +278,23 @@ func TestUpdateWithNilLabelsOnLease(t *testing.T) { t.Fatalf("Failed to get lease: %v", err) } - leaseLock.lease.Labels = nil - - leaseLock.Labels = map[string]string{"custom-key": "custom-val"} + leaseLock.lease.Labels = map[string]string{"custom-key": "custom-val"} - // Update should succeed even with nil Labels on the lease itself - if err := leaseLock.Update(context.Background(), testRecord); err != nil { - t.Errorf("Update failed with nil Labels: %v", err) + // Update labels + lease, err := leaseLock.Client.Leases(testNamespace).Update(context.Background(), leaseLock.lease, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Failed to update lease labels: %v", err) } -} -func TestUpdateWithNilLabelsOnLeaseLock(t *testing.T) { - setup() - - // Create initial lease - if err := leaseLock.Create(context.Background(), testRecord); err != nil { - t.Fatalf("Failed to create lease: %v", err) + val, exists := lease.Labels["custom-key"] + if !exists { + t.Error("Label was overidden on the lease") } - // Get the lease to initialize leaseLock.lease - if _, _, err := leaseLock.Get(context.Background()); err != nil { - t.Fatalf("Failed to get lease: %v", err) + if val != "custom-val" { + t.Errorf("Label value mismatch, got %q want %q", val, "custom-val") } - leaseLock.Labels = nil - - leaseLock.lease.Labels = map[string]string{"custom-key": "custom-val"} - - // Update should succeed even with nil Labels on the leaselock + // Update should succeed even with nil Labels if err := leaseLock.Update(context.Background(), testRecord); err != nil { t.Errorf("Update failed with nil Labels: %v", err) } diff --git a/staging/src/k8s.io/client-go/util/cert/cert.go b/staging/src/k8s.io/client-go/util/cert/cert.go index 48c78b595ef38..1220461264c65 100644 --- a/staging/src/k8s.io/client-go/util/cert/cert.go +++ b/staging/src/k8s.io/client-go/util/cert/cert.go @@ -75,15 +75,13 @@ func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, erro CommonName: cfg.CommonName, Organization: cfg.Organization, }, + DNSNames: []string{cfg.CommonName}, NotBefore: notBefore, NotAfter: now.Add(duration365d * 10).UTC(), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, BasicConstraintsValid: true, IsCA: true, } - if len(cfg.CommonName) > 0 { - tmpl.DNSNames = []string{cfg.CommonName} - } certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key) if err != nil { diff --git a/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/allocatortesting/allocator_testing.go b/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/allocatortesting/allocator_testing.go index f145e76c5ad1f..dcf2bf02b51f1 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/allocatortesting/allocator_testing.go +++ b/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/allocatortesting/allocator_testing.go @@ -87,7 +87,6 @@ const ( claim1 = "claim-1" slice1 = "slice-1" slice2 = "slice-2" - device0 = "device-0" device1 = "device-1" device2 = "device-2" device3 = "device-3" @@ -1990,51 +1989,12 @@ func TestAllocator(t *testing.T, node: node(node1, region1), expectResults: []any{ - allocationResultWithConfigs( - localNodeSelector(node1), - objects(deviceAllocationResult(req0, driverA, pool1, device1, false)), - []resourceapi.DeviceAllocationConfiguration{ - { - Source: resourceapi.AllocationConfigSourceClass, - Requests: []string{req0}, - DeviceConfiguration: deviceConfiguration(driverA, "classAttribute"), - }, - }, - ), - }, - }, - "with-class-device-config-with-multiple-request-and-configs": { - claimsToAllocate: objects(claimWithRequests(claim0, []resourceapi.DeviceConstraint{}, request(req0, classA, 1), request(req1, classA, 1), request(req2, classB, 1))), - classes: objects( - classWithConfig(classA, driverA, "classAttribute-A"), - classWithConfig(classB, driverB, "classAttribute-B"), - ), - slices: unwrap( - sliceWithMultipleDevices(slice1, node1, pool1, driverA, 2), - sliceWithOneDevice(slice2, node1, pool1, driverB), - ), - node: node(node1, region1), - - expectResults: []any{ - allocationResultWithConfigs( + allocationResultWithConfig( localNodeSelector(node1), - objects( - deviceAllocationResult(req0, driverA, pool1, device0, false), - deviceAllocationResult(req1, driverA, pool1, device1, false), - deviceAllocationResult(req2, driverB, pool1, device1, false), - ), - []resourceapi.DeviceAllocationConfiguration{ - { - Source: resourceapi.AllocationConfigSourceClass, - Requests: []string{req0, req1}, - DeviceConfiguration: deviceConfiguration(driverA, "classAttribute-A"), - }, - { - Source: resourceapi.AllocationConfigSourceClass, - Requests: []string{req2}, - DeviceConfiguration: deviceConfiguration(driverB, "classAttribute-B"), - }, - }, + driverA, + resourceapi.AllocationConfigSourceClass, + "classAttribute", + deviceAllocationResult(req0, driverA, pool1, device1, false), ), }, }, @@ -2285,7 +2245,7 @@ func TestAllocator(t *testing.T, []resourceapi.DeviceAllocationConfiguration{ { Source: resourceapi.AllocationConfigSourceClass, - Requests: []string{req0SubReq1}, + Requests: nil, DeviceConfiguration: deviceConfiguration(driverB, "bar"), }, }, diff --git a/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/experimental/allocator_experimental.go b/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/experimental/allocator_experimental.go index 6b3c69b791641..84218307c6be8 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/experimental/allocator_experimental.go +++ b/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/experimental/allocator_experimental.go @@ -388,11 +388,6 @@ func (a *Allocator) Allocate(ctx context.Context, node *v1.Node, claims []*resou } // Populate configs. - - // Each class config gets added only once. - // We need to keep track of which class configs have already been added and at which position in the allocationResult.Devices.Config. - type configRange struct{ start, end int } - configIndexesForClass := make(map[string]configRange) // Key: class name / Value: position of the configs for the class in allocationResult.Devices.Config. for requestIndex := range claim.Spec.Devices.Requests { requestKey := requestIndices{claimIndex: claimIndex, requestIndex: requestIndex} requestData := alloc.requestData[requestKey] @@ -403,29 +398,15 @@ func (a *Allocator) Allocate(ctx context.Context, node *v1.Node, claims []*resou } class := requestData.class - if class == nil { - continue - } - configIndexes, exists := configIndexesForClass[class.Name] - if exists { - // The configs for the class have already been added. - // Just append the request name for the request class. - for i := configIndexes.start; i < configIndexes.end; i++ { - allocationResult.Devices.Config[i].Requests = append(allocationResult.Devices.Config[i].Requests, requestData.requestName()) + if class != nil { + for _, config := range class.Spec.Config { + allocationResult.Devices.Config = append(allocationResult.Devices.Config, resourceapi.DeviceAllocationConfiguration{ + Source: resourceapi.AllocationConfigSourceClass, + Requests: nil, // All of them... + DeviceConfiguration: config.DeviceConfiguration, + }) } - continue } - - // Add all configs for the class once. - initialConfigLen := len(allocationResult.Devices.Config) - for _, config := range class.Spec.Config { - allocationResult.Devices.Config = append(allocationResult.Devices.Config, resourceapi.DeviceAllocationConfiguration{ - Source: resourceapi.AllocationConfigSourceClass, - Requests: []string{requestData.requestName()}, - DeviceConfiguration: config.DeviceConfiguration, - }) - } - configIndexesForClass[class.Name] = configRange{start: initialConfigLen, end: len(allocationResult.Devices.Config)} } for _, config := range claim.Spec.Devices.Config { // If Requests are empty, it applies to all. So it can just be included. @@ -665,13 +646,6 @@ type requestData struct { allDevices []deviceWithID } -func (rd *requestData) requestName() string { - if rd.parentRequest != nil { - return fmt.Sprintf("%s/%s", rd.parentRequest.name(), rd.request.name()) - } - return rd.request.name() -} - type deviceWithID struct { *draapi.Device id DeviceID diff --git a/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/incubating/allocator_incubating.go b/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/incubating/allocator_incubating.go index b3b4f7d2c662d..d65235e35b016 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/incubating/allocator_incubating.go +++ b/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/incubating/allocator_incubating.go @@ -295,11 +295,6 @@ func (a *Allocator) Allocate(ctx context.Context, node *v1.Node, claims []*resou } // Populate configs. - - // Each class config gets added only once. - // We need to keep track of which class configs have already been added and at which position in the allocationResult.Devices.Config. - type configRange struct{ start, end int } - configIndexesForClass := make(map[string]configRange) // Key: class name / Value: position of the configs for the class in allocationResult.Devices.Config. for requestIndex := range claim.Spec.Devices.Requests { requestKey := requestIndices{claimIndex: claimIndex, requestIndex: requestIndex} requestData := alloc.requestData[requestKey] @@ -310,29 +305,15 @@ func (a *Allocator) Allocate(ctx context.Context, node *v1.Node, claims []*resou } class := requestData.class - if class == nil { - continue - } - configIndexes, exists := configIndexesForClass[class.Name] - if exists { - // The configs for the class have already been added. - // Just append the request name for the request class. - for i := configIndexes.start; i < configIndexes.end; i++ { - allocationResult.Devices.Config[i].Requests = append(allocationResult.Devices.Config[i].Requests, requestData.requestName()) + if class != nil { + for _, config := range class.Spec.Config { + allocationResult.Devices.Config = append(allocationResult.Devices.Config, resourceapi.DeviceAllocationConfiguration{ + Source: resourceapi.AllocationConfigSourceClass, + Requests: nil, // All of them... + DeviceConfiguration: config.DeviceConfiguration, + }) } - continue } - - // Add all configs for the class once. - initialConfigLen := len(allocationResult.Devices.Config) - for _, config := range class.Spec.Config { - allocationResult.Devices.Config = append(allocationResult.Devices.Config, resourceapi.DeviceAllocationConfiguration{ - Source: resourceapi.AllocationConfigSourceClass, - Requests: []string{requestData.requestName()}, - DeviceConfiguration: config.DeviceConfiguration, - }) - } - configIndexesForClass[class.Name] = configRange{start: initialConfigLen, end: len(allocationResult.Devices.Config)} } for _, config := range claim.Spec.Devices.Config { // If Requests are empty, it applies to all. So it can just be included. @@ -554,13 +535,6 @@ type requestData struct { allDevices []deviceWithID } -func (rd *requestData) requestName() string { - if rd.parentRequest != nil { - return fmt.Sprintf("%s/%s", rd.parentRequest.name(), rd.request.name()) - } - return rd.request.name() -} - type deviceWithID struct { *draapi.Device id DeviceID diff --git a/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/stable/allocator_stable.go b/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/stable/allocator_stable.go index 54440746b784f..d5b118e931e1e 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/stable/allocator_stable.go +++ b/staging/src/k8s.io/dynamic-resource-allocation/structured/internal/stable/allocator_stable.go @@ -286,11 +286,6 @@ func (a *Allocator) Allocate(ctx context.Context, node *v1.Node, claims []*resou } // Populate configs. - - // Each class config gets added only once. - // We need to keep track of which class configs have already been added and at which position in the allocationResult.Devices.Config. - type configRange struct{ start, end int } - configIndexesForClass := make(map[string]configRange) // Key: class name / Value: position of the configs for the class in allocationResult.Devices.Config. for requestIndex := range claim.Spec.Devices.Requests { requestKey := requestIndices{claimIndex: claimIndex, requestIndex: requestIndex} requestData := alloc.requestData[requestKey] @@ -301,29 +296,15 @@ func (a *Allocator) Allocate(ctx context.Context, node *v1.Node, claims []*resou } class := requestData.class - if class == nil { - continue - } - configIndexes, exists := configIndexesForClass[class.Name] - if exists { - // The configs for the class have already been added. - // Just append the request name for the request class. - for i := configIndexes.start; i < configIndexes.end; i++ { - allocationResult.Devices.Config[i].Requests = append(allocationResult.Devices.Config[i].Requests, requestData.requestName()) + if class != nil { + for _, config := range class.Spec.Config { + allocationResult.Devices.Config = append(allocationResult.Devices.Config, resourceapi.DeviceAllocationConfiguration{ + Source: resourceapi.AllocationConfigSourceClass, + Requests: nil, // All of them... + DeviceConfiguration: config.DeviceConfiguration, + }) } - continue } - - // Add all configs for the class once. - initialConfigLen := len(allocationResult.Devices.Config) - for _, config := range class.Spec.Config { - allocationResult.Devices.Config = append(allocationResult.Devices.Config, resourceapi.DeviceAllocationConfiguration{ - Source: resourceapi.AllocationConfigSourceClass, - Requests: []string{requestData.requestName()}, - DeviceConfiguration: config.DeviceConfiguration, - }) - } - configIndexesForClass[class.Name] = configRange{start: initialConfigLen, end: len(allocationResult.Devices.Config)} } for _, config := range claim.Spec.Devices.Config { // If Requests are empty, it applies to all. So it can just be included. @@ -536,13 +517,6 @@ type requestData struct { allDevices []deviceWithID } -func (rd *requestData) requestName() string { - if rd.parentRequest != nil { - return fmt.Sprintf("%s/%s", rd.parentRequest.name(), rd.request.name()) - } - return rd.request.name() -} - type deviceWithID struct { *draapi.Device id DeviceID diff --git a/staging/src/k8s.io/kubectl/pkg/cmd/apiresources/apiresources.go b/staging/src/k8s.io/kubectl/pkg/cmd/apiresources/apiresources.go index 9a45cd907d2e6..1959cc1109d26 100644 --- a/staging/src/k8s.io/kubectl/pkg/cmd/apiresources/apiresources.go +++ b/staging/src/k8s.io/kubectl/pkg/cmd/apiresources/apiresources.go @@ -229,9 +229,6 @@ func (o *APIResourceOptions) RunAPIResources() error { allResources = append(allResources, apiList) } - if len(allResources) == 0 { - return utilerrors.NewAggregate(errs) - } flatList := &metav1.APIResourceList{ TypeMeta: metav1.TypeMeta{ APIVersion: allResources[0].APIVersion, diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index b1e225f9a8825..fd3ca8715fce5 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -1417,7 +1417,7 @@ version: "1.29" - name: SchedulerAsyncAPICalls versionedSpecs: - - default: false + - default: true lockToDefault: false preRelease: Beta version: "1.34" @@ -1633,12 +1633,6 @@ lockToDefault: true preRelease: GA version: "1.32" -- name: StatefulSetSemanticRevisionComparison - versionedSpecs: - - default: true - lockToDefault: false - preRelease: Beta - version: "1.0" - name: StorageCapacityScoring versionedSpecs: - default: false diff --git a/test/conformance/image/Makefile b/test/conformance/image/Makefile index 61a963f6352ca..17f84faa2ff27 100644 --- a/test/conformance/image/Makefile +++ b/test/conformance/image/Makefile @@ -33,7 +33,7 @@ CLUSTER_DIR?=$(shell pwd)/../../../cluster/ # This is defined in root Makefile, but some build contexts do not refer to them KUBE_BASE_IMAGE_REGISTRY?=registry.k8s.io -BASE_IMAGE_VERSION?=bookworm-v1.0.6 +BASE_IMAGE_VERSION?=bookworm-v1.0.4 RUNNERIMAGE?=${KUBE_BASE_IMAGE_REGISTRY}/build-image/debian-base-${ARCH}:${BASE_IMAGE_VERSION} TEMP_DIR:=$(shell mktemp -d -t conformance-XXXXXX) diff --git a/test/e2e/common/node/framework/cgroups/cgroups_linux.go b/test/e2e/common/node/framework/cgroups/cgroups_linux.go index c7f3064b5080d..c77c560d50f84 100644 --- a/test/e2e/common/node/framework/cgroups/cgroups_linux.go +++ b/test/e2e/common/node/framework/cgroups/cgroups_linux.go @@ -17,33 +17,13 @@ limitations under the License. package cgroups import ( - "math" "strconv" + libcontainercgroups "github.com/opencontainers/cgroups" v1 "k8s.io/api/core/v1" kubecm "k8s.io/kubernetes/pkg/kubelet/cm" ) -// convertCPUSharesToCgroupV2Value is copied from ConvertCPUSharesToCgroupV2Value in opencontainers/cgroups. -// https://github.com/opencontainers/cgroups/pull/20/files -func convertCPUSharesToCgroupV2Value(cpuShares uint64) uint64 { - // The value of 0 means "unset". - if cpuShares == 0 { - return 0 - } - if cpuShares <= 2 { - return 1 - } - if cpuShares >= 262144 { - return 10000 - } - l := math.Log2(float64(cpuShares)) - // Quadratic function which fits min, max, and default. - exponent := (l*l+125*l)/612.0 - 7.0/34.0 - - return uint64(math.Ceil(math.Pow(10, exponent))) -} - func getExpectedCPUShares(rr *v1.ResourceRequirements, podOnCgroupv2 bool) []string { // This function is moved out from cgroups.go because opencontainers/cgroups can only be compiled in linux platforms. cpuRequest := rr.Requests.Cpu() @@ -62,7 +42,7 @@ func getExpectedCPUShares(rr *v1.ResourceRequirements, podOnCgroupv2 bool) []str // container runtimes, we check if either the old or the new conversion matches the actual value for now. // TODO: Remove the old conversion once container runtimes are updated. oldConverted := 1 + ((shares-2)*9999)/262142 - converted := convertCPUSharesToCgroupV2Value(uint64(shares)) + converted := libcontainercgroups.ConvertCPUSharesToCgroupV2Value(uint64(shares)) return []string{strconv.FormatInt(oldConverted, 10), strconv.FormatInt(int64(converted), 10)} } else { return []string{strconv.FormatInt(shares, 10)} diff --git a/test/e2e/framework/pv/wait.go b/test/e2e/framework/pv/wait.go index aa94b17081bb8..ebfe227afe37a 100644 --- a/test/e2e/framework/pv/wait.go +++ b/test/e2e/framework/pv/wait.go @@ -76,16 +76,11 @@ func WaitForPersistentVolumeClaimModificationFailure(ctx context.Context, c clie desiredClass := ptr.Deref(claim.Spec.VolumeAttributesClassName, "") var match = func(claim *v1.PersistentVolumeClaim) bool { - foundErrorCondition := false for _, condition := range claim.Status.Conditions { - if condition.Type == v1.PersistentVolumeClaimVolumeModifyVolumeError { - foundErrorCondition = true + if condition.Type != v1.PersistentVolumeClaimVolumeModifyVolumeError { + return false } } - // if no error found it must be an error - if !foundErrorCondition { - return false - } // check if claim's current volume attributes class is NOT desired one, and has appropriate ModifyVolumeStatus currentClass := ptr.Deref(claim.Status.CurrentVolumeAttributesClassName, "") diff --git a/test/e2e_node/container_lifecycle_test.go b/test/e2e_node/container_lifecycle_test.go index 6c99596dc7814..8362c3000bf8e 100644 --- a/test/e2e_node/container_lifecycle_test.go +++ b/test/e2e_node/container_lifecycle_test.go @@ -5906,113 +5906,7 @@ var _ = SIGDescribe(feature.SidecarContainers, "Containers Lifecycle", func() { }) }) }) - }) - - ginkgo.When("A restartable init container with startup probe fails initially", func() { - ginkgo.It("should continue probing and allow regular container to start after the restartable init container recovers", func(ctx context.Context) { - restartableInit := "buggy-restartable-init" - regularContainer := "regular-container" - - restartPolicyAlways := v1.ContainerRestartPolicyAlways - - pod := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "restartable-init-startup-probe-fix", - }, - Spec: v1.PodSpec{ - RestartPolicy: v1.RestartPolicyNever, - InitContainers: []v1.Container{{ - Name: restartableInit, - Image: busyboxImage, - RestartPolicy: &restartPolicyAlways, - Command: []string{"sh", "-c", ` -if [ ! -f /persistent/first_run_done ]; then - echo 'First run: creating marker and exiting with 1' - touch /persistent/first_run_done - exit 1 -else - echo 'Second run: marker found, running as sidecar' - sleep 120 -fi`}, - StartupProbe: &v1.Probe{ - InitialDelaySeconds: 3, - PeriodSeconds: 2, - FailureThreshold: 10, - ProbeHandler: v1.ProbeHandler{ - Exec: &v1.ExecAction{ - Command: []string{"/bin/true"}, - }, - }, - }, - }}, - Containers: []v1.Container{{ - Name: regularContainer, - Image: imageutils.GetPauseImageName(), - StartupProbe: &v1.Probe{ - InitialDelaySeconds: 5, - PeriodSeconds: 5, - ProbeHandler: v1.ProbeHandler{ - Exec: &v1.ExecAction{ - Command: []string{"/bin/true"}, - }, - }, - }, - }}, - }, - } - - preparePod(pod) - client := e2epod.NewPodClient(f) - pod = client.Create(ctx, pod) - - ginkgo.By("Waiting for init container to fail and restart at least once") - framework.ExpectNoError( - e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, - "restartable init restarted", 90*time.Second, - func(p *v1.Pod) (bool, error) { - for _, st := range p.Status.InitContainerStatuses { - if st.Name == restartableInit && st.RestartCount > 0 { - framework.Logf("Init container %s has restarted %d times", restartableInit, st.RestartCount) - return true, nil - } - } - return false, nil - }), - ) - - ginkgo.By("Waiting for init container to be running after restart") - framework.ExpectNoError( - e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, - "restartable init running", 90*time.Second, - func(p *v1.Pod) (bool, error) { - for _, st := range p.Status.InitContainerStatuses { - if st.Name == restartableInit && st.State.Running != nil { - framework.Logf("Init container %s is now running", restartableInit) - return true, nil - } - } - return false, nil - }), - ) - - ginkgo.By("Waiting for regular container to start") - framework.ExpectNoError( - e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, - "regular container running", 120*time.Second, - func(p *v1.Pod) (bool, error) { - for _, st := range p.Status.ContainerStatuses { - if st.Name == regularContainer && st.State.Running != nil { - framework.Logf("Regular container %s is running", regularContainer) - return true, nil - } - } - return false, nil - }), - ) - }) - }) - }) var _ = SIGDescribe(feature.SidecarContainers, framework.WithSerial(), "Containers Lifecycle", func() { diff --git a/test/images/nonroot/BASEIMAGE b/test/images/nonroot/BASEIMAGE index b0a0270b001af..5c683ef4c68a9 100644 --- a/test/images/nonroot/BASEIMAGE +++ b/test/images/nonroot/BASEIMAGE @@ -1,7 +1,7 @@ -linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.6 -linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.6 -linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.6 -linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.6 -linux/s390x=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.6 +linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.4 +linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.4 +linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.4 +linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.4 +linux/s390x=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.4 windows/amd64/1809=mcr.microsoft.com/windows/nanoserver:1809 windows/amd64/ltsc2022=mcr.microsoft.com/windows/nanoserver:ltsc2022 diff --git a/test/images/pets/peer-finder/BASEIMAGE b/test/images/pets/peer-finder/BASEIMAGE index 8b900d2f419b1..880445d9c10b4 100644 --- a/test/images/pets/peer-finder/BASEIMAGE +++ b/test/images/pets/peer-finder/BASEIMAGE @@ -1,5 +1,5 @@ -linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.6 -linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.6 -linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.6 -linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.6 -linux/s390x=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.6 +linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.4 +linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.4 +linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.4 +linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.4 +linux/s390x=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.4 diff --git a/test/images/pets/zookeeper-installer/BASEIMAGE b/test/images/pets/zookeeper-installer/BASEIMAGE index afc96c3e3017d..1e453bde46eed 100644 --- a/test/images/pets/zookeeper-installer/BASEIMAGE +++ b/test/images/pets/zookeeper-installer/BASEIMAGE @@ -1,4 +1,4 @@ -linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.6 -linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.6 -linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.6 -linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.6 +linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.4 +linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.4 +linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.4 +linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.4 diff --git a/test/images/regression-issue-74839/BASEIMAGE b/test/images/regression-issue-74839/BASEIMAGE index 8b900d2f419b1..880445d9c10b4 100644 --- a/test/images/regression-issue-74839/BASEIMAGE +++ b/test/images/regression-issue-74839/BASEIMAGE @@ -1,5 +1,5 @@ -linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.6 -linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.6 -linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.6 -linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.6 -linux/s390x=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.6 +linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.4 +linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.4 +linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.4 +linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.4 +linux/s390x=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.4 diff --git a/test/images/resource-consumer/BASEIMAGE b/test/images/resource-consumer/BASEIMAGE index b0a0270b001af..5c683ef4c68a9 100644 --- a/test/images/resource-consumer/BASEIMAGE +++ b/test/images/resource-consumer/BASEIMAGE @@ -1,7 +1,7 @@ -linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.6 -linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.6 -linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.6 -linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.6 -linux/s390x=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.6 +linux/amd64=registry.k8s.io/build-image/debian-base-amd64:bookworm-v1.0.4 +linux/arm=registry.k8s.io/build-image/debian-base-arm:bookworm-v1.0.4 +linux/arm64=registry.k8s.io/build-image/debian-base-arm64:bookworm-v1.0.4 +linux/ppc64le=registry.k8s.io/build-image/debian-base-ppc64le:bookworm-v1.0.4 +linux/s390x=registry.k8s.io/build-image/debian-base-s390x:bookworm-v1.0.4 windows/amd64/1809=mcr.microsoft.com/windows/nanoserver:1809 windows/amd64/ltsc2022=mcr.microsoft.com/windows/nanoserver:ltsc2022 diff --git a/test/integration/apimachinery/apply/apply_test.go b/test/integration/apimachinery/apply/apply_test.go index 0d6caed004c8e..ce51b5299a53a 100644 --- a/test/integration/apimachinery/apply/apply_test.go +++ b/test/integration/apimachinery/apply/apply_test.go @@ -83,20 +83,12 @@ func testOptionalListMapKey(tCtx ktesting.TContext) { requireManagedFields := func(what string, obj *unstructured.Unstructured, expectedManagedFields any) { tCtx.Helper() actualManagedFields, _, _ := unstructured.NestedFieldCopy(obj.Object, "metadata", "managedFields") + // Strip non-deterministic time. if actualManagedFields != nil { managers := actualManagedFields.([]any) for i := range managers { - // Strip non-deterministic time. unstructured.RemoveNestedField(managers[i].(map[string]any), "time") } - // Sort by manager. There should be at most one entry per manager, so - // no need for a tie breaker. Semantically the order is irrelevant, - // so we don't need to expect a specific one here. - // - // The order turned out to be non-deterministic (test flake!) without this. - slices.SortFunc(managers, func(a, b any) int { - return strings.Compare(a.(map[string]any)["manager"].(string), b.(map[string]any)["manager"].(string)) - }) } require.Equal(tCtx, dump(expectedManagedFields), dump(actualManagedFields), "%s:\n%s", what, dump(obj)) } diff --git a/test/integration/auth/auth_test.go b/test/integration/auth/auth_test.go index d80b049d07e6f..3641b7e4aef12 100644 --- a/test/integration/auth/auth_test.go +++ b/test/integration/auth/auth_test.go @@ -41,9 +41,10 @@ import ( "testing" "time" + utiltesting "k8s.io/client-go/util/testing" + "github.com/google/go-cmp/cmp" - authenticationv1 "k8s.io/api/authentication/v1" authenticationv1beta1 "k8s.io/api/authentication/v1beta1" certificatesv1 "k8s.io/api/certificates/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -64,7 +65,6 @@ import ( "k8s.io/client-go/rest" v1 "k8s.io/client-go/tools/clientcmd/api/v1" resttransport "k8s.io/client-go/transport" - utiltesting "k8s.io/client-go/util/testing" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/pkg/apis/autoscaling" @@ -1567,48 +1567,3 @@ func newTestWebhookTokenAuthServer() *httptest.Server { server.Start() return server } - -func TestSloppySANCertificates(t *testing.T) { - tCtx := ktesting.Init(t) - _, kubeConfig, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{ - ModifyServerRunOptions: func(opts *options.ServerRunOptions) { - // append to opts.Authentication.ClientCert.ClientCA - fmt.Println(opts.Authentication.ClientCert.ClientCA) - caData, err := os.ReadFile(opts.Authentication.ClientCert.ClientCA) - if err != nil { - t.Fatal(err) - } - sloppyCAData, err := os.ReadFile("testdata/sloppy-san-root.pem") - if err != nil { - t.Fatal(err) - } - err = os.WriteFile(opts.Authentication.ClientCert.ClientCA, []byte(string(caData)+"\n"+string(sloppyCAData)), os.FileMode(0644)) - if err != nil { - t.Fatal(err) - } - }, - }) - defer tearDownFn() - - var err error - kubeConfig = rest.AnonymousClientConfig(kubeConfig) - kubeConfig.CertData, err = os.ReadFile("testdata/sloppy-san-client.pem") - if err != nil { - t.Fatal(err) - } - kubeConfig.KeyData, err = os.ReadFile("testdata/sloppy-san-client-key.pem") - if err != nil { - t.Fatal(err) - } - c, err := clientset.NewForConfig(kubeConfig) - if err != nil { - t.Fatal(err) - } - r, err := c.AuthenticationV1().SelfSubjectReviews().Create(tCtx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{}) - if err != nil { - t.Fatal(err) - } - if r.Status.UserInfo.Username != "sloppy-san-client" { - t.Fatalf("expected sloppy-san-client, got %#v", r.Status.UserInfo) - } -} diff --git a/test/integration/auth/testdata/README.md b/test/integration/auth/testdata/README.md deleted file mode 100644 index a78ddfbd05a3f..0000000000000 --- a/test/integration/auth/testdata/README.md +++ /dev/null @@ -1 +0,0 @@ -Keys in this directory are generated for testing purposes only. diff --git a/test/integration/auth/testdata/sloppy-san-client-key.pem b/test/integration/auth/testdata/sloppy-san-client-key.pem deleted file mode 100644 index b6d0a3a7c896c..0000000000000 --- a/test/integration/auth/testdata/sloppy-san-client-key.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEILndj5ixo79V24VqIlvSS0J5rwQyklP4+D+rAAjO763loAoGCCqGSM49 -AwEHoUQDQgAEXMW9sg8iyagwwhlJ94p0brB5NqSYIytoj18bs6xPZ3UqEZo5BhXi -2m2Cx8althrzoXbMIGr+ALUWDgKk7BVuSg== ------END EC PRIVATE KEY----- diff --git a/test/integration/auth/testdata/sloppy-san-client.pem b/test/integration/auth/testdata/sloppy-san-client.pem deleted file mode 100644 index 8627574055129..0000000000000 --- a/test/integration/auth/testdata/sloppy-san-client.pem +++ /dev/null @@ -1,65 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 6084561304482469822 (0x5470b444dfc3d7be) - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN=sloppy-san-root - Validity - Not Before: Oct 14 00:20:19 2025 GMT - Not After : Sep 20 00:20:19 2125 GMT - Subject: CN=sloppy-san-client - Subject Public Key Info: - Public Key Algorithm: id-ecPublicKey - Public-Key: (256 bit) - pub: - 04:5c:c5:bd:b2:0f:22:c9:a8:30:c2:19:49:f7:8a: - 74:6e:b0:79:36:a4:98:23:2b:68:8f:5f:1b:b3:ac: - 4f:67:75:2a:11:9a:39:06:15:e2:da:6d:82:c7:c6: - a5:b6:1a:f3:a1:76:cc:20:6a:fe:00:b5:16:0e:02: - a4:ec:15:6e:4a - ASN1 OID: prime256v1 - NIST CURVE: P-256 - X509v3 extensions: - X509v3 Key Usage: critical - Digital Signature, Key Encipherment - X509v3 Extended Key Usage: - TLS Web Client Authentication - X509v3 Basic Constraints: critical - CA:FALSE - X509v3 Authority Key Identifier: - 9A:A1:A8:3C:30:1B:EC:1F:B2:1F:10:0E:0C:42:A8:2A:B8:97:9A:8E - X509v3 Subject Alternative Name: - DNS:, DNS:example.com., email:not-an-email - Signature Algorithm: sha256WithRSAEncryption - Signature Value: - 96:bc:48:3c:aa:f6:8c:e4:a4:b5:40:6d:fe:20:1b:60:40:12: - e9:f5:58:94:0e:0d:dc:6d:a3:83:ae:3d:05:3b:64:1a:f4:c0: - 23:c9:0d:63:02:ea:c2:f4:e8:bc:88:20:8e:2e:bb:f0:79:32: - cc:0a:59:e2:17:6f:63:aa:5e:b8:0d:54:15:2f:5c:eb:08:7e: - eb:fe:31:62:b1:e5:da:88:dc:be:9e:20:01:f1:73:40:8d:13: - 55:36:aa:2e:58:13:b4:85:aa:63:30:2c:47:a4:95:61:33:f3: - 31:c7:f8:91:d1:18:3c:65:a8:fb:a4:8f:dc:51:8e:9a:d5:dc: - eb:04:b5:b1:f9:82:f5:ff:4a:7b:27:b3:3e:8e:59:30:93:57: - 7d:f2:b4:af:94:39:2b:b9:0d:c1:e5:94:0f:8d:83:03:74:e5: - 6f:38:cd:ee:df:1b:5d:64:48:b9:05:27:5f:09:12:c8:03:96: - 36:0d:d4:19:5b:be:76:ea:7d:f3:20:08:2a:b4:c6:92:63:41: - 44:d8:2c:b1:b8:71:7c:a9:1f:26:d7:99:04:d4:9b:a6:4b:a4: - fa:ef:b2:a9:f5:e7:af:53:4b:de:00:45:5c:5b:f0:2a:1a:bc: - 40:2f:97:ca:fb:9c:53:a8:16:46:89:a0:f9:43:45:47:de:3e: - 09:8e:a6:22 ------BEGIN CERTIFICATE----- -MIICbjCCAVagAwIBAgIIVHC0RN/D174wDQYJKoZIhvcNAQELBQAwGjEYMBYGA1UE -AxMPc2xvcHB5LXNhbi1yb290MCAXDTI1MTAxNDAwMjAxOVoYDzIxMjUwOTIwMDAy -MDE5WjAcMRowGAYDVQQDExFzbG9wcHktc2FuLWNsaWVudDBZMBMGByqGSM49AgEG -CCqGSM49AwEHA0IABFzFvbIPIsmoMMIZSfeKdG6weTakmCMraI9fG7OsT2d1KhGa -OQYV4tptgsfGpbYa86F2zCBq/gC1Fg4CpOwVbkqjfzB9MA4GA1UdDwEB/wQEAwIF -oDATBgNVHSUEDDAKBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB8GA1UdIwQYMBaA -FJqhqDwwG+wfsh8QDgxCqCq4l5qOMCcGA1UdEQQgMB6CAIIMZXhhbXBsZS5jb20u -gQxub3QtYW4tZW1haWwwDQYJKoZIhvcNAQELBQADggEBAJa8SDyq9ozkpLVAbf4g -G2BAEun1WJQODdxto4OuPQU7ZBr0wCPJDWMC6sL06LyIII4uu/B5MswKWeIXb2Oq -XrgNVBUvXOsIfuv+MWKx5dqI3L6eIAHxc0CNE1U2qi5YE7SFqmMwLEeklWEz8zHH -+JHRGDxlqPukj9xRjprV3OsEtbH5gvX/Snsnsz6OWTCTV33ytK+UOSu5DcHllA+N -gwN05W84ze7fG11kSLkFJ18JEsgDljYN1BlbvnbqffMgCCq0xpJjQUTYLLG4cXyp -HybXmQTUm6ZLpPrvsqn1569TS94ARVxb8CoavEAvl8r7nFOoFkaJoPlDRUfePgmO -piI= ------END CERTIFICATE----- diff --git a/test/integration/auth/testdata/sloppy-san-root.pem b/test/integration/auth/testdata/sloppy-san-root.pem deleted file mode 100644 index 54386e1487a3b..0000000000000 --- a/test/integration/auth/testdata/sloppy-san-root.pem +++ /dev/null @@ -1,78 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 2937139693522916239 (0x28c2d0fd5822138f) - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN=sloppy-san-root - Validity - Not Before: Oct 14 00:20:19 2025 GMT - Not After : Sep 20 00:20:19 2125 GMT - Subject: CN=sloppy-san-root - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:9e:00:38:70:8d:a7:9c:fd:89:e4:33:f9:7e:e3: - 99:1f:19:69:a9:a3:7c:ca:8e:f4:52:ef:c8:a8:3d: - fc:6c:08:95:8b:14:f9:d9:a2:2b:25:7c:15:8e:aa: - a9:ec:19:fe:62:9f:18:33:12:72:b0:2f:37:b6:de: - d0:24:fe:19:ef:78:93:b7:7d:7a:44:46:4c:14:bd: - d7:23:a7:fc:44:43:6d:f9:29:f8:79:2a:61:fc:1f: - d4:79:49:19:53:5b:6d:5a:66:cd:59:a9:2b:38:c7: - c5:38:96:b8:12:36:c5:60:d8:dc:ea:86:df:9a:cd: - 50:95:be:5f:1a:38:67:dc:bf:67:24:5e:ed:06:79: - 32:b5:19:bd:11:ec:ff:61:b7:e2:32:05:8d:b6:c9: - 12:ba:92:7c:2a:9e:26:71:b2:d0:85:95:9d:68:79: - d0:3e:e5:8b:ac:e8:e4:22:6d:79:a3:77:58:01:72: - f9:67:7d:d8:5e:7f:5c:56:45:31:36:8e:f5:be:48: - c4:66:f1:14:ed:38:43:ae:5f:cc:20:66:7b:48:df: - 78:d5:f4:4f:67:2a:d4:ee:7b:36:d2:c1:5f:d1:3b: - e4:bb:31:0f:94:0c:19:f7:17:99:99:04:eb:b7:b4: - 34:6c:f9:0b:8c:61:e9:a5:5b:50:62:f7:24:51:25: - 3d:43 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Key Usage: critical - Digital Signature, Key Encipherment, Certificate Sign - X509v3 Basic Constraints: critical - CA:TRUE - X509v3 Subject Key Identifier: - 9A:A1:A8:3C:30:1B:EC:1F:B2:1F:10:0E:0C:42:A8:2A:B8:97:9A:8E - X509v3 Subject Alternative Name: - DNS:, DNS:example.com. - Signature Algorithm: sha256WithRSAEncryption - Signature Value: - 4d:4f:ef:47:a0:41:96:9a:e0:98:e3:e6:5e:4f:70:6a:b1:16: - 3c:10:8b:f9:2b:12:57:58:28:88:a4:1a:e8:4c:a4:be:a0:c1: - ad:07:95:dd:d9:bc:db:a9:db:31:5f:42:30:60:19:e2:28:fb: - 72:78:91:a7:83:e7:bd:0f:52:b8:2b:fe:d0:0e:03:64:0e:08: - 8b:62:b9:bc:30:1d:76:86:42:a6:fe:f0:55:0d:3c:16:97:32: - 3a:9f:1a:0e:5b:01:68:9d:37:76:d5:ed:a8:e5:e6:1b:7d:ff: - b2:e3:c0:a0:8f:cb:2f:98:e5:6b:e5:b6:ef:fe:a4:c4:f8:33: - 6f:e1:90:89:16:69:58:c8:ca:95:99:d1:84:8e:0e:83:ed:a7: - ae:ac:4e:32:7e:72:95:fa:ce:3f:62:ae:06:57:40:b2:bf:79: - 8f:b2:f6:69:07:ee:d8:7c:70:b0:52:8d:f6:08:f9:de:a8:6a: - 90:77:6a:65:52:67:82:98:32:68:66:4d:8e:6b:a8:dd:b5:3c: - a7:fe:b4:98:d0:69:70:1b:60:60:1b:10:30:88:5c:9b:f0:6b: - 9e:52:47:2f:83:7d:77:e3:e2:af:a5:fb:de:65:91:51:0f:27: - b2:34:25:8f:97:55:ee:11:d0:d1:4e:8f:7a:cf:9f:7d:8e:e6: - 27:24:61:cf ------BEGIN CERTIFICATE----- -MIIDFTCCAf2gAwIBAgIIKMLQ/VgiE48wDQYJKoZIhvcNAQELBQAwGjEYMBYGA1UE -AxMPc2xvcHB5LXNhbi1yb290MCAXDTI1MTAxNDAwMjAxOVoYDzIxMjUwOTIwMDAy -MDE5WjAaMRgwFgYDVQQDEw9zbG9wcHktc2FuLXJvb3QwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQCeADhwjaec/YnkM/l+45kfGWmpo3zKjvRS78ioPfxs -CJWLFPnZoislfBWOqqnsGf5inxgzEnKwLze23tAk/hnveJO3fXpERkwUvdcjp/xE -Q235Kfh5KmH8H9R5SRlTW21aZs1ZqSs4x8U4lrgSNsVg2Nzqht+azVCVvl8aOGfc -v2ckXu0GeTK1Gb0R7P9ht+IyBY22yRK6knwqniZxstCFlZ1oedA+5Yus6OQibXmj -d1gBcvlnfdhef1xWRTE2jvW+SMRm8RTtOEOuX8wgZntI33jV9E9nKtTuezbSwV/R -O+S7MQ+UDBn3F5mZBOu3tDRs+QuMYemlW1Bi9yRRJT1DAgMBAAGjXTBbMA4GA1Ud -DwEB/wQEAwICpDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSaoag8MBvsH7If -EA4MQqgquJeajjAZBgNVHREEEjAQggCCDGV4YW1wbGUuY29tLjANBgkqhkiG9w0B -AQsFAAOCAQEATU/vR6BBlprgmOPmXk9warEWPBCL+SsSV1goiKQa6EykvqDBrQeV -3dm826nbMV9CMGAZ4ij7cniRp4PnvQ9SuCv+0A4DZA4Ii2K5vDAddoZCpv7wVQ08 -FpcyOp8aDlsBaJ03dtXtqOXmG33/suPAoI/LL5jla+W27/6kxPgzb+GQiRZpWMjK -lZnRhI4Og+2nrqxOMn5ylfrOP2KuBldAsr95j7L2aQfu2HxwsFKN9gj53qhqkHdq -ZVJngpgyaGZNjmuo3bU8p/60mNBpcBtgYBsQMIhcm/BrnlJHL4N9d+Pir6X73mWR -UQ8nsjQlj5dV7hHQ0U6Pes+ffY7mJyRhzw== ------END CERTIFICATE----- diff --git a/test/integration/job/job_test.go b/test/integration/job/job_test.go index 3e0bee1d264e1..8e8e2aa9ef437 100644 --- a/test/integration/job/job_test.go +++ b/test/integration/job/job_test.go @@ -4091,85 +4091,6 @@ func TestSuspendJob(t *testing.T) { } } -// TestStartTimeUpdateOnResume verifies that the job controller can update startTime -// when resuming a suspended job (https://github.com/kubernetes/kubernetes/issues/134521). -func TestStartTimeUpdateOnResume(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobManagedBy, true) - - closeFn, restConfig, clientSet, ns := setup(t, "suspend-starttime-validation") - t.Cleanup(closeFn) - ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig) - t.Cleanup(cancel) - - job, err := createJobWithDefaults(ctx, clientSet, ns.Name, &batchv1.Job{ - Spec: batchv1.JobSpec{ - Parallelism: ptr.To[int32](1), - Completions: ptr.To[int32](2), - Suspend: ptr.To(false), - }, - }) - if err != nil { - t.Fatalf("Failed to create Job: %v", err) - } - - validateJobsPodsStatusOnly(ctx, t, clientSet, job, podsByStatus{ - Active: 1, - Ready: ptr.To[int32](0), - Terminating: ptr.To[int32](0), - }) - - job, err = clientSet.BatchV1().Jobs(ns.Name).Get(ctx, job.Name, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Failed to get Job: %v", err) - } - if job.Status.StartTime == nil { - t.Fatalf("Job startTime was not set") - } - - job.Spec.Suspend = ptr.To(true) - job, err = clientSet.BatchV1().Jobs(ns.Name).Update(ctx, job, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("Failed to suspend Job: %v", err) - } - - validateJobsPodsStatusOnly(ctx, t, clientSet, job, podsByStatus{ - Active: 0, - Ready: ptr.To[int32](0), - Terminating: ptr.To[int32](0), - }) - - job, err = clientSet.BatchV1().Jobs(ns.Name).Get(ctx, job.Name, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Failed to get Job: %v", err) - } - if getJobConditionStatus(ctx, job, batchv1.JobSuspended) != v1.ConditionTrue { - t.Fatalf("JobSuspended condition was not set to True") - } - - job.Spec.Suspend = ptr.To(false) - job, err = clientSet.BatchV1().Jobs(ns.Name).Update(ctx, job, metav1.UpdateOptions{}) - if err != nil { - t.Fatalf("Failed to resume Job: %v", err) - } - - validateJobsPodsStatusOnly(ctx, t, clientSet, job, podsByStatus{ - Active: 1, - Ready: ptr.To[int32](0), - Terminating: ptr.To[int32](0), - }) - - job, err = clientSet.BatchV1().Jobs(ns.Name).Get(ctx, job.Name, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Failed to get Job: %v", err) - } - if getJobConditionStatus(ctx, job, batchv1.JobSuspended) != v1.ConditionFalse { - t.Error("JobSuspended condition was not set to False") - } - if job.Status.StartTime == nil { - t.Error("Job startTime was not set after resume") - } -} - // TestSuspendJobWithZeroCompletions verifies the suspended Job with // completions=0 is marked as Complete. func TestSuspendJobWithZeroCompletions(t *testing.T) { diff --git a/test/integration/scheduler/preemption/misc/main_test.go b/test/integration/scheduler/preemption/misc/main_test.go deleted file mode 100644 index 3dd88c34eb5c2..0000000000000 --- a/test/integration/scheduler/preemption/misc/main_test.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package miscpreemption - -import ( - "testing" - - "k8s.io/kubernetes/test/integration/framework" -) - -func TestMain(m *testing.M) { - framework.EtcdMain(m.Run) -} diff --git a/test/integration/scheduler/preemption/misc/miscpreemption_test.go b/test/integration/scheduler/preemption/misc/miscpreemption_test.go deleted file mode 100644 index 805ecd104af6f..0000000000000 --- a/test/integration/scheduler/preemption/misc/miscpreemption_test.go +++ /dev/null @@ -1,1238 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package miscpreemption - -import ( - "fmt" - "testing" - "time" - - v1 "k8s.io/api/core/v1" - policy "k8s.io/api/policy/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/wait" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/informers" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - featuregatetesting "k8s.io/component-base/featuregate/testing" - "k8s.io/component-helpers/storage/volume" - "k8s.io/klog/v2" - configv1 "k8s.io/kube-scheduler/config/v1" - podutil "k8s.io/kubernetes/pkg/api/v1/pod" - "k8s.io/kubernetes/pkg/apis/scheduling" - "k8s.io/kubernetes/pkg/features" - "k8s.io/kubernetes/pkg/scheduler" - configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing" - "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumerestrictions" - st "k8s.io/kubernetes/pkg/scheduler/testing" - "k8s.io/kubernetes/plugin/pkg/admission/priority" - testutils "k8s.io/kubernetes/test/integration/util" - "k8s.io/utils/ptr" -) - -// imported from testutils -var ( - initPausePod = testutils.InitPausePod - createNode = testutils.CreateNode - createPausePod = testutils.CreatePausePod - runPausePod = testutils.RunPausePod - initTest = testutils.InitTestSchedulerWithNS - initTestDisablePreemption = testutils.InitTestDisablePreemption - initDisruptionController = testutils.InitDisruptionController - waitCachedPodsStable = testutils.WaitCachedPodsStable - podIsGettingEvicted = testutils.PodIsGettingEvicted - podUnschedulable = testutils.PodUnschedulable - waitForPDBsStable = testutils.WaitForPDBsStable - waitForPodToScheduleWithTimeout = testutils.WaitForPodToScheduleWithTimeout - waitForPodUnschedulable = testutils.WaitForPodUnschedulable -) - -var lowPriority, mediumPriority, highPriority = int32(100), int32(200), int32(300) - -// TestNonPreemption tests NonPreempt option of PriorityClass of scheduler works as expected. -func TestNonPreemption(t *testing.T) { - var preemptNever = v1.PreemptNever - // Initialize scheduler. - testCtx := initTest(t, "non-preemption") - cs := testCtx.ClientSet - tests := []struct { - name string - PreemptionPolicy *v1.PreemptionPolicy - }{ - { - name: "pod preemption will happen", - PreemptionPolicy: nil, - }, - { - name: "pod preemption will not happen", - PreemptionPolicy: &preemptNever, - }, - } - victim := initPausePod(&testutils.PausePodConfig{ - Name: "victim-pod", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(400, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, - }, - }) - - preemptor := initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(300, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, - }, - }) - - // Create a node with some resources - nodeRes := map[v1.ResourceName]string{ - v1.ResourcePods: "32", - v1.ResourceCPU: "500m", - v1.ResourceMemory: "500", - } - _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) - if err != nil { - t.Fatalf("Error creating nodes: %v", err) - } - - for _, asyncPreemptionEnabled := range []bool{true, false} { - for _, test := range tests { - t.Run(fmt.Sprintf("%s (Async preemption enabled: %v)", test.name, asyncPreemptionEnabled), func(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) - - defer testutils.CleanupPods(testCtx.Ctx, cs, t, []*v1.Pod{preemptor, victim}) - preemptor.Spec.PreemptionPolicy = test.PreemptionPolicy - victimPod, err := createPausePod(cs, victim) - if err != nil { - t.Fatalf("Error while creating victim: %v", err) - } - if err := waitForPodToScheduleWithTimeout(testCtx.Ctx, cs, victimPod, 5*time.Second); err != nil { - t.Fatalf("victim %v should be become scheduled", victimPod.Name) - } - - preemptorPod, err := createPausePod(cs, preemptor) - if err != nil { - t.Fatalf("Error while creating preemptor: %v", err) - } - - err = testutils.WaitForNominatedNodeNameWithTimeout(testCtx.Ctx, cs, preemptorPod, 5*time.Second) - // test.PreemptionPolicy == nil means we expect the preemptor to be nominated. - expect := test.PreemptionPolicy == nil - // err == nil indicates the preemptor is indeed nominated. - got := err == nil - if got != expect { - t.Errorf("Expect preemptor to be nominated=%v, but got=%v", expect, got) - } - }) - } - } -} - -// TestDisablePreemption tests disable pod preemption of scheduler works as expected. -func TestDisablePreemption(t *testing.T) { - // Initialize scheduler, and disable preemption. - testCtx := initTestDisablePreemption(t, "disable-preemption") - cs := testCtx.ClientSet - - tests := []struct { - name string - existingPods []*v1.Pod - pod *v1.Pod - }{ - { - name: "pod preemption will not happen", - existingPods: []*v1.Pod{ - initPausePod(&testutils.PausePodConfig{ - Name: "victim-pod", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(400, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, - }, - }), - }, - pod: initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(300, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, - }, - }), - }, - } - - // Create a node with some resources - nodeRes := map[v1.ResourceName]string{ - v1.ResourcePods: "32", - v1.ResourceCPU: "500m", - v1.ResourceMemory: "500", - } - _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) - if err != nil { - t.Fatalf("Error creating nodes: %v", err) - } - - for _, asyncPreemptionEnabled := range []bool{true, false} { - for _, test := range tests { - t.Run(fmt.Sprintf("%s (Async preemption enabled: %v)", test.name, asyncPreemptionEnabled), func(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) - - pods := make([]*v1.Pod, len(test.existingPods)) - // Create and run existingPods. - for i, p := range test.existingPods { - pods[i], err = runPausePod(cs, p) - if err != nil { - t.Fatalf("Test [%v]: Error running pause pod: %v", test.name, err) - } - } - // Create the "pod". - preemptor, err := createPausePod(cs, test.pod) - if err != nil { - t.Errorf("Error while creating high priority pod: %v", err) - } - // Ensure preemptor should keep unschedulable. - if err := waitForPodUnschedulable(testCtx.Ctx, cs, preemptor); err != nil { - t.Errorf("Preemptor %v should not become scheduled", preemptor.Name) - } - - // Ensure preemptor should not be nominated. - if err := testutils.WaitForNominatedNodeNameWithTimeout(testCtx.Ctx, cs, preemptor, 5*time.Second); err == nil { - t.Errorf("Preemptor %v should not be nominated", preemptor.Name) - } - - // Cleanup - pods = append(pods, preemptor) - testutils.CleanupPods(testCtx.Ctx, cs, t, pods) - }) - } - } -} - -// This test verifies that system critical priorities are created automatically and resolved properly. -func TestPodPriorityResolution(t *testing.T) { - admission := priority.NewPlugin() - testCtx := testutils.InitTestScheduler(t, testutils.InitTestAPIServer(t, "preemption", admission)) - cs := testCtx.ClientSet - - // Build clientset and informers for controllers. - externalClientConfig := restclient.CopyConfig(testCtx.KubeConfig) - externalClientConfig.QPS = -1 - externalClientset := clientset.NewForConfigOrDie(externalClientConfig) - externalInformers := informers.NewSharedInformerFactory(externalClientset, time.Second) - admission.SetExternalKubeClientSet(externalClientset) - admission.SetExternalKubeInformerFactory(externalInformers) - - // Waiting for all controllers to sync - testutils.SyncSchedulerInformerFactory(testCtx) - externalInformers.Start(testCtx.Ctx.Done()) - externalInformers.WaitForCacheSync(testCtx.Ctx.Done()) - - // Run all controllers - go testCtx.Scheduler.Run(testCtx.Ctx) - - tests := []struct { - Name string - PriorityClass string - Pod *v1.Pod - ExpectedPriority int32 - ExpectedError error - }{ - { - Name: "SystemNodeCritical priority class", - PriorityClass: scheduling.SystemNodeCritical, - ExpectedPriority: scheduling.SystemCriticalPriority + 1000, - Pod: initPausePod(&testutils.PausePodConfig{ - Name: fmt.Sprintf("pod1-%v", scheduling.SystemNodeCritical), - Namespace: metav1.NamespaceSystem, - PriorityClassName: scheduling.SystemNodeCritical, - }), - }, - { - Name: "SystemClusterCritical priority class", - PriorityClass: scheduling.SystemClusterCritical, - ExpectedPriority: scheduling.SystemCriticalPriority, - Pod: initPausePod(&testutils.PausePodConfig{ - Name: fmt.Sprintf("pod2-%v", scheduling.SystemClusterCritical), - Namespace: metav1.NamespaceSystem, - PriorityClassName: scheduling.SystemClusterCritical, - }), - }, - { - Name: "Invalid priority class should result in error", - PriorityClass: "foo", - ExpectedPriority: scheduling.SystemCriticalPriority, - Pod: initPausePod(&testutils.PausePodConfig{ - Name: fmt.Sprintf("pod3-%v", scheduling.SystemClusterCritical), - Namespace: metav1.NamespaceSystem, - PriorityClassName: "foo", - }), - ExpectedError: fmt.Errorf("failed to create pause pod: pods \"pod3-system-cluster-critical\" is forbidden: no PriorityClass with name foo was found"), - }, - } - - // Create a node with some resources - nodeRes := map[v1.ResourceName]string{ - v1.ResourcePods: "32", - v1.ResourceCPU: "500m", - v1.ResourceMemory: "500", - } - _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) - if err != nil { - t.Fatalf("Error creating nodes: %v", err) - } - - pods := make([]*v1.Pod, 0, len(tests)) - for _, asyncPreemptionEnabled := range []bool{true, false} { - for _, test := range tests { - t.Run(fmt.Sprintf("%s (Async preemption enabled: %v)", test.Name, asyncPreemptionEnabled), func(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) - - pod, err := runPausePod(cs, test.Pod) - if err != nil { - if test.ExpectedError == nil { - t.Fatalf("Test [PodPriority/%v]: Error running pause pod: %v", test.PriorityClass, err) - } - if err.Error() != test.ExpectedError.Error() { - t.Fatalf("Test [PodPriority/%v]: Expected error %v but got error %v", test.PriorityClass, test.ExpectedError, err) - } - return - } - pods = append(pods, pod) - if pod.Spec.Priority != nil { - if *pod.Spec.Priority != test.ExpectedPriority { - t.Errorf("Expected pod %v to have priority %v but was %v", pod.Name, test.ExpectedPriority, pod.Spec.Priority) - } - } else { - t.Errorf("Expected pod %v to have priority %v but was nil", pod.Name, test.PriorityClass) - } - testutils.CleanupPods(testCtx.Ctx, cs, t, pods) - }) - } - } - testutils.CleanupNodes(cs, t) -} - -func mkPriorityPodWithGrace(tc *testutils.TestContext, name string, priority int32, grace int64) *v1.Pod { - defaultPodRes := &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(100, resource.DecimalSI)}, - } - pod := initPausePod(&testutils.PausePodConfig{ - Name: name, - Namespace: tc.NS.Name, - Priority: &priority, - Labels: map[string]string{"pod": name}, - Resources: defaultPodRes, - }) - pod.Spec.TerminationGracePeriodSeconds = &grace - return pod -} - -// This test ensures that while the preempting pod is waiting for the victims to -// terminate, other pending lower priority pods are not scheduled in the room created -// after preemption and while the higher priority pods is not scheduled yet. -func TestPreemptionStarvation(t *testing.T) { - // Initialize scheduler. - testCtx := initTest(t, "preemption") - cs := testCtx.ClientSet - - tests := []struct { - name string - numExistingPod int - numExpectedPending int - preemptor *v1.Pod - }{ - { - // This test ensures that while the preempting pod is waiting for the victims - // terminate, other lower priority pods are not scheduled in the room created - // after preemption and while the higher priority pods is not scheduled yet. - name: "starvation test: higher priority pod is scheduled before the lower priority ones", - numExistingPod: 10, - numExpectedPending: 5, - preemptor: initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(300, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, - }, - }), - }, - } - - // Create a node with some resources - nodeRes := map[v1.ResourceName]string{ - v1.ResourcePods: "32", - v1.ResourceCPU: "500m", - v1.ResourceMemory: "500", - } - _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) - if err != nil { - t.Fatalf("Error creating nodes: %v", err) - } - - for _, asyncPreemptionEnabled := range []bool{true, false} { - for _, clearingNominatedNodeNameAfterBinding := range []bool{true, false} { - for _, test := range tests { - t.Run(fmt.Sprintf("%s (Async preemption enabled: %v, ClearingNominatedNodeNameAfterBinding: %v)", test.name, asyncPreemptionEnabled, clearingNominatedNodeNameAfterBinding), func(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ClearingNominatedNodeNameAfterBinding, clearingNominatedNodeNameAfterBinding) - - pendingPods := make([]*v1.Pod, test.numExpectedPending) - numRunningPods := test.numExistingPod - test.numExpectedPending - runningPods := make([]*v1.Pod, numRunningPods) - // Create and run existingPods. - for i := 0; i < numRunningPods; i++ { - runningPods[i], err = createPausePod(cs, mkPriorityPodWithGrace(testCtx, fmt.Sprintf("rpod-%v", i), mediumPriority, 0)) - if err != nil { - t.Fatalf("Error creating pause pod: %v", err) - } - } - // make sure that runningPods are all scheduled. - for _, p := range runningPods { - if err := testutils.WaitForPodToSchedule(testCtx.Ctx, cs, p); err != nil { - t.Fatalf("Pod %v/%v didn't get scheduled: %v", p.Namespace, p.Name, err) - } - } - // Create pending pods. - for i := 0; i < test.numExpectedPending; i++ { - pendingPods[i], err = createPausePod(cs, mkPriorityPodWithGrace(testCtx, fmt.Sprintf("ppod-%v", i), mediumPriority, 0)) - if err != nil { - t.Fatalf("Error creating pending pod: %v", err) - } - } - // Make sure that all pending pods are being marked unschedulable. - for _, p := range pendingPods { - if err := wait.PollUntilContextTimeout(testCtx.Ctx, 100*time.Millisecond, wait.ForeverTestTimeout, false, - podUnschedulable(cs, p.Namespace, p.Name)); err != nil { - t.Errorf("Pod %v/%v didn't get marked unschedulable: %v", p.Namespace, p.Name, err) - } - } - // Create the preemptor. - preemptor, err := createPausePod(cs, test.preemptor) - if err != nil { - t.Errorf("Error while creating the preempting pod: %v", err) - } - - // Make sure that preemptor is scheduled after preemptions. - if err := testutils.WaitForPodToScheduleWithTimeout(testCtx.Ctx, cs, preemptor, 60*time.Second); err != nil { - t.Errorf("Preemptor pod %v didn't get scheduled: %v", preemptor.Name, err) - } - - // Check if .status.nominatedNodeName of the preemptor pod gets set when feature gate is disabled. - // This test always expects preemption to occur since numExistingPod (10) fills the node completely. - if !clearingNominatedNodeNameAfterBinding { - if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil { - t.Errorf(".status.nominatedNodeName was not set for pod %v/%v: %v", preemptor.Namespace, preemptor.Name, err) - } - } - // Cleanup - klog.Info("Cleaning up all pods...") - allPods := pendingPods - allPods = append(allPods, runningPods...) - allPods = append(allPods, preemptor) - testutils.CleanupPods(testCtx.Ctx, cs, t, allPods) - }) - } - } - } -} - -// TestPreemptionRaces tests that other scheduling events and operations do not -// race with the preemption process. -func TestPreemptionRaces(t *testing.T) { - // Initialize scheduler. - testCtx := initTest(t, "preemption-race") - cs := testCtx.ClientSet - - tests := []struct { - name string - numInitialPods int // Pods created and executed before running preemptor - numAdditionalPods int // Pods created after creating the preemptor - numRepetitions int // Repeat the tests to check races - preemptor *v1.Pod - }{ - { - // This test ensures that while the preempting pod is waiting for the victims - // terminate, other lower priority pods are not scheduled in the room created - // after preemption and while the higher priority pods is not scheduled yet. - name: "ensures that other pods are not scheduled while preemptor is being marked as nominated (issue #72124)", - numInitialPods: 2, - numAdditionalPods: 20, - numRepetitions: 5, - preemptor: initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(4900, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(4900, resource.DecimalSI)}, - }, - }), - }, - } - - // Create a node with some resources - nodeRes := map[v1.ResourceName]string{ - v1.ResourcePods: "100", - v1.ResourceCPU: "5000m", - v1.ResourceMemory: "5000", - } - _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) - if err != nil { - t.Fatalf("Error creating nodes: %v", err) - } - - for _, asyncPreemptionEnabled := range []bool{true, false} { - for _, clearingNominatedNodeNameAfterBinding := range []bool{true, false} { - for _, test := range tests { - t.Run(fmt.Sprintf("%s (Async preemption enabled: %v, ClearingNominatedNodeNameAfterBinding: %v)", test.name, asyncPreemptionEnabled, clearingNominatedNodeNameAfterBinding), func(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ClearingNominatedNodeNameAfterBinding, clearingNominatedNodeNameAfterBinding) - - if test.numRepetitions <= 0 { - test.numRepetitions = 1 - } - for n := 0; n < test.numRepetitions; n++ { - initialPods := make([]*v1.Pod, test.numInitialPods) - additionalPods := make([]*v1.Pod, test.numAdditionalPods) - // Create and run existingPods. - for i := 0; i < test.numInitialPods; i++ { - initialPods[i], err = createPausePod(cs, mkPriorityPodWithGrace(testCtx, fmt.Sprintf("rpod-%v", i), mediumPriority, 0)) - if err != nil { - t.Fatalf("Error creating pause pod: %v", err) - } - } - // make sure that initial Pods are all scheduled. - for _, p := range initialPods { - if err := testutils.WaitForPodToSchedule(testCtx.Ctx, cs, p); err != nil { - t.Fatalf("Pod %v/%v didn't get scheduled: %v", p.Namespace, p.Name, err) - } - } - // Create the preemptor. - klog.Info("Creating the preemptor pod...") - preemptor, err := createPausePod(cs, test.preemptor) - if err != nil { - t.Errorf("Error while creating the preempting pod: %v", err) - } - - klog.Info("Creating additional pods...") - for i := 0; i < test.numAdditionalPods; i++ { - additionalPods[i], err = createPausePod(cs, mkPriorityPodWithGrace(testCtx, fmt.Sprintf("ppod-%v", i), mediumPriority, 0)) - if err != nil { - t.Fatalf("Error creating pending pod: %v", err) - } - } - // Make sure that preemptor is scheduled after preemptions. - if err := testutils.WaitForPodToScheduleWithTimeout(testCtx.Ctx, cs, preemptor, 60*time.Second); err != nil { - t.Errorf("Preemptor pod %v didn't get scheduled: %v", preemptor.Name, err) - } - - // Check that the preemptor pod gets nominated node name when feature gate is disabled. - if !clearingNominatedNodeNameAfterBinding { - if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil { - t.Errorf(".status.nominatedNodeName was not set for pod %v/%v: %v", preemptor.Namespace, preemptor.Name, err) - } - } - - klog.Info("Check unschedulable pods still exists and were never scheduled...") - for _, p := range additionalPods { - pod, err := cs.CoreV1().Pods(p.Namespace).Get(testCtx.Ctx, p.Name, metav1.GetOptions{}) - if err != nil { - t.Errorf("Error in getting Pod %v/%v info: %v", p.Namespace, p.Name, err) - } - if len(pod.Spec.NodeName) > 0 { - t.Errorf("Pod %v/%v is already scheduled", p.Namespace, p.Name) - } - _, cond := podutil.GetPodCondition(&pod.Status, v1.PodScheduled) - if cond != nil && cond.Status != v1.ConditionFalse { - t.Errorf("Pod %v/%v is no longer unschedulable: %v", p.Namespace, p.Name, err) - } - } - // Cleanup - klog.Info("Cleaning up all pods...") - allPods := additionalPods - allPods = append(allPods, initialPods...) - allPods = append(allPods, preemptor) - testutils.CleanupPods(testCtx.Ctx, cs, t, allPods) - } - }) - } - } - } -} - -func mkMinAvailablePDB(name, namespace string, uid types.UID, minAvailable int, matchLabels map[string]string) *policy.PodDisruptionBudget { - intMinAvailable := intstr.FromInt32(int32(minAvailable)) - return &policy.PodDisruptionBudget{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: policy.PodDisruptionBudgetSpec{ - MinAvailable: &intMinAvailable, - Selector: &metav1.LabelSelector{MatchLabels: matchLabels}, - }, - } -} - -func addPodConditionReady(pod *v1.Pod) { - pod.Status = v1.PodStatus{ - Phase: v1.PodRunning, - Conditions: []v1.PodCondition{ - { - Type: v1.PodReady, - Status: v1.ConditionTrue, - }, - }, - } -} - -// TestPDBInPreemption tests PodDisruptionBudget support in preemption. -func TestPDBInPreemption(t *testing.T) { - // Initialize scheduler. - testCtx := initTest(t, "preemption-pdb") - cs := testCtx.ClientSet - - initDisruptionController(t, testCtx) - - defaultPodRes := &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(100, resource.DecimalSI)}, - } - defaultNodeRes := map[v1.ResourceName]string{ - v1.ResourcePods: "32", - v1.ResourceCPU: "500m", - v1.ResourceMemory: "500", - } - - tests := []struct { - name string - nodeCnt int - pdbs []*policy.PodDisruptionBudget - pdbPodNum []int32 - existingPods []*v1.Pod - pod *v1.Pod - preemptedPodIndexes map[int]struct{} - }{ - { - name: "A non-PDB violating pod is preempted despite its higher priority", - nodeCnt: 1, - pdbs: []*policy.PodDisruptionBudget{ - mkMinAvailablePDB("pdb-1", testCtx.NS.Name, types.UID("pdb-1-uid"), 2, map[string]string{"foo": "bar"}), - }, - pdbPodNum: []int32{2}, - existingPods: []*v1.Pod{ - initPausePod(&testutils.PausePodConfig{ - Name: "low-pod1", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: defaultPodRes, - Labels: map[string]string{"foo": "bar"}, - }), - initPausePod(&testutils.PausePodConfig{ - Name: "low-pod2", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: defaultPodRes, - Labels: map[string]string{"foo": "bar"}, - }), - initPausePod(&testutils.PausePodConfig{ - Name: "mid-pod3", - Namespace: testCtx.NS.Name, - Priority: &mediumPriority, - Resources: defaultPodRes, - }), - }, - pod: initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(300, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, - }, - }), - preemptedPodIndexes: map[int]struct{}{2: {}}, - }, - { - name: "A node without any PDB violating pods is preferred for preemption", - nodeCnt: 2, - pdbs: []*policy.PodDisruptionBudget{ - mkMinAvailablePDB("pdb-1", testCtx.NS.Name, types.UID("pdb-1-uid"), 2, map[string]string{"foo": "bar"}), - }, - pdbPodNum: []int32{1}, - existingPods: []*v1.Pod{ - initPausePod(&testutils.PausePodConfig{ - Name: "low-pod1", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: defaultPodRes, - NodeName: "node-1", - Labels: map[string]string{"foo": "bar"}, - }), - initPausePod(&testutils.PausePodConfig{ - Name: "mid-pod2", - Namespace: testCtx.NS.Name, - Priority: &mediumPriority, - NodeName: "node-2", - Resources: defaultPodRes, - }), - }, - pod: initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, - }, - }), - preemptedPodIndexes: map[int]struct{}{1: {}}, - }, - { - name: "A node with fewer PDB violating pods is preferred for preemption", - nodeCnt: 3, - pdbs: []*policy.PodDisruptionBudget{ - mkMinAvailablePDB("pdb-1", testCtx.NS.Name, types.UID("pdb-1-uid"), 2, map[string]string{"foo1": "bar"}), - mkMinAvailablePDB("pdb-2", testCtx.NS.Name, types.UID("pdb-2-uid"), 2, map[string]string{"foo2": "bar"}), - }, - pdbPodNum: []int32{1, 5}, - existingPods: []*v1.Pod{ - initPausePod(&testutils.PausePodConfig{ - Name: "low-pod1", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: defaultPodRes, - NodeName: "node-1", - Labels: map[string]string{"foo1": "bar"}, - }), - initPausePod(&testutils.PausePodConfig{ - Name: "mid-pod1", - Namespace: testCtx.NS.Name, - Priority: &mediumPriority, - Resources: defaultPodRes, - NodeName: "node-1", - }), - initPausePod(&testutils.PausePodConfig{ - Name: "low-pod2", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: defaultPodRes, - NodeName: "node-2", - Labels: map[string]string{"foo2": "bar"}, - }), - initPausePod(&testutils.PausePodConfig{ - Name: "mid-pod2", - Namespace: testCtx.NS.Name, - Priority: &mediumPriority, - Resources: defaultPodRes, - NodeName: "node-2", - Labels: map[string]string{"foo2": "bar"}, - }), - initPausePod(&testutils.PausePodConfig{ - Name: "low-pod4", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: defaultPodRes, - NodeName: "node-3", - Labels: map[string]string{"foo2": "bar"}, - }), - initPausePod(&testutils.PausePodConfig{ - Name: "low-pod5", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: defaultPodRes, - NodeName: "node-3", - Labels: map[string]string{"foo2": "bar"}, - }), - initPausePod(&testutils.PausePodConfig{ - Name: "low-pod6", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Resources: defaultPodRes, - NodeName: "node-3", - Labels: map[string]string{"foo2": "bar"}, - }), - }, - pod: initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(400, resource.DecimalSI)}, - }, - }), - // The third node is chosen because PDB is not violated for node 3 and the victims have lower priority than node-2. - preemptedPodIndexes: map[int]struct{}{4: {}, 5: {}, 6: {}}, - }, - } - - for _, asyncPreemptionEnabled := range []bool{true, false} { - for _, clearingNominatedNodeNameAfterBinding := range []bool{true, false} { - for _, test := range tests { - t.Run(fmt.Sprintf("%s (Async preemption enabled: %v, ClearingNominatedNodeNameAfterBinding: %v)", test.name, asyncPreemptionEnabled, clearingNominatedNodeNameAfterBinding), func(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ClearingNominatedNodeNameAfterBinding, clearingNominatedNodeNameAfterBinding) - - for i := 1; i <= test.nodeCnt; i++ { - nodeName := fmt.Sprintf("node-%v", i) - _, err := createNode(cs, st.MakeNode().Name(nodeName).Capacity(defaultNodeRes).Obj()) - if err != nil { - t.Fatalf("Error creating node %v: %v", nodeName, err) - } - } - - pods := make([]*v1.Pod, len(test.existingPods)) - var err error - // Create and run existingPods. - for i, p := range test.existingPods { - if pods[i], err = runPausePod(cs, p); err != nil { - t.Fatalf("Test [%v]: Error running pause pod: %v", test.name, err) - } - // Add pod condition ready so that PDB is updated. - addPodConditionReady(p) - if _, err := testCtx.ClientSet.CoreV1().Pods(testCtx.NS.Name).UpdateStatus(testCtx.Ctx, p, metav1.UpdateOptions{}); err != nil { - t.Fatal(err) - } - } - // Wait for Pods to be stable in scheduler cache. - if err := waitCachedPodsStable(testCtx, test.existingPods); err != nil { - t.Fatalf("Not all pods are stable in the cache: %v", err) - } - - // Create PDBs. - for _, pdb := range test.pdbs { - _, err := testCtx.ClientSet.PolicyV1().PodDisruptionBudgets(testCtx.NS.Name).Create(testCtx.Ctx, pdb, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("Failed to create PDB: %v", err) - } - } - // Wait for PDBs to become stable. - if err := waitForPDBsStable(testCtx, test.pdbs, test.pdbPodNum); err != nil { - t.Fatalf("Not all pdbs are stable in the cache: %v", err) - } - - // Create the "pod". - preemptor, err := createPausePod(cs, test.pod) - if err != nil { - t.Errorf("Error while creating high priority pod: %v", err) - } - // Wait for preemption of pods and make sure the other ones are not preempted. - for i, p := range pods { - if _, found := test.preemptedPodIndexes[i]; found { - if err = wait.PollUntilContextTimeout(testCtx.Ctx, time.Second, wait.ForeverTestTimeout, false, - podIsGettingEvicted(cs, p.Namespace, p.Name)); err != nil { - t.Errorf("Test [%v]: Pod %v/%v is not getting evicted.", test.name, p.Namespace, p.Name) - } - } else { - if p.DeletionTimestamp != nil { - t.Errorf("Test [%v]: Didn't expect pod %v/%v to get preempted.", test.name, p.Namespace, p.Name) - } - } - } - // Also check if .status.nominatedNodeName of the preemptor pod gets set. - if len(test.preemptedPodIndexes) > 0 && !clearingNominatedNodeNameAfterBinding { - if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil { - t.Errorf("Test [%v]: .status.nominatedNodeName was not set for pod %v/%v: %v", test.name, preemptor.Namespace, preemptor.Name, err) - } - } - - // Cleanup - pods = append(pods, preemptor) - testutils.CleanupPods(testCtx.Ctx, cs, t, pods) - if err := cs.PolicyV1().PodDisruptionBudgets(testCtx.NS.Name).DeleteCollection(testCtx.Ctx, metav1.DeleteOptions{}, metav1.ListOptions{}); err != nil { - t.Errorf("error while deleting PDBs, error: %v", err) - } - if err := cs.CoreV1().Nodes().DeleteCollection(testCtx.Ctx, metav1.DeleteOptions{}, metav1.ListOptions{}); err != nil { - t.Errorf("error whiling deleting nodes, error: %v", err) - } - }) - } - } - } -} - -// TestReadWriteOncePodPreemption tests preemption scenarios for pods with -// ReadWriteOncePod PVCs. -func TestReadWriteOncePodPreemption(t *testing.T) { - cfg := configtesting.V1ToInternalWithDefaults(t, configv1.KubeSchedulerConfiguration{ - Profiles: []configv1.KubeSchedulerProfile{{ - SchedulerName: ptr.To(v1.DefaultSchedulerName), - Plugins: &configv1.Plugins{ - Filter: configv1.PluginSet{ - Enabled: []configv1.Plugin{ - {Name: volumerestrictions.Name}, - }, - }, - PreFilter: configv1.PluginSet{ - Enabled: []configv1.Plugin{ - {Name: volumerestrictions.Name}, - }, - }, - }, - }}, - }) - - testCtx := testutils.InitTestSchedulerWithOptions(t, - testutils.InitTestAPIServer(t, "preemption", nil), - 0, - scheduler.WithProfiles(cfg.Profiles...)) - testutils.SyncSchedulerInformerFactory(testCtx) - go testCtx.Scheduler.Run(testCtx.Ctx) - - cs := testCtx.ClientSet - - storage := v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}} - volType := v1.HostPathDirectoryOrCreate - pv1 := st.MakePersistentVolume(). - Name("pv-with-read-write-once-pod-1"). - AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). - Capacity(storage.Requests). - HostPathVolumeSource(&v1.HostPathVolumeSource{Path: "/mnt1", Type: &volType}). - Obj() - pvc1 := st.MakePersistentVolumeClaim(). - Name("pvc-with-read-write-once-pod-1"). - Namespace(testCtx.NS.Name). - // Annotation and volume name required for PVC to be considered bound. - Annotation(volume.AnnBindCompleted, "true"). - VolumeName(pv1.Name). - AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). - Resources(storage). - Obj() - pv2 := st.MakePersistentVolume(). - Name("pv-with-read-write-once-pod-2"). - AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). - Capacity(storage.Requests). - HostPathVolumeSource(&v1.HostPathVolumeSource{Path: "/mnt2", Type: &volType}). - Obj() - pvc2 := st.MakePersistentVolumeClaim(). - Name("pvc-with-read-write-once-pod-2"). - Namespace(testCtx.NS.Name). - // Annotation and volume name required for PVC to be considered bound. - Annotation(volume.AnnBindCompleted, "true"). - VolumeName(pv2.Name). - AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). - Resources(storage). - Obj() - - tests := []struct { - name string - init func() error - existingPods []*v1.Pod - pod *v1.Pod - unresolvable bool - preemptedPodIndexes map[int]struct{} - cleanup func() error - }{ - { - name: "preempt single pod", - init: func() error { - _, err := testutils.CreatePV(cs, pv1) - if err != nil { - return fmt.Errorf("cannot create pv: %w", err) - } - _, err = testutils.CreatePVC(cs, pvc1) - if err != nil { - return fmt.Errorf("cannot create pvc: %w", err) - } - return nil - }, - existingPods: []*v1.Pod{ - initPausePod(&testutils.PausePodConfig{ - Name: "victim-pod", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Volumes: []v1.Volume{{ - Name: "volume", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc1.Name, - }, - }, - }}, - }), - }, - pod: initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Volumes: []v1.Volume{{ - Name: "volume", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc1.Name, - }, - }, - }}, - }), - preemptedPodIndexes: map[int]struct{}{0: {}}, - cleanup: func() error { - if err := testutils.DeletePVC(cs, pvc1.Name, pvc1.Namespace); err != nil { - return fmt.Errorf("cannot delete pvc: %w", err) - } - if err := testutils.DeletePV(cs, pv1.Name); err != nil { - return fmt.Errorf("cannot delete pv: %w", err) - } - return nil - }, - }, - { - name: "preempt two pods", - init: func() error { - for _, pv := range []*v1.PersistentVolume{pv1, pv2} { - _, err := testutils.CreatePV(cs, pv) - if err != nil { - return fmt.Errorf("cannot create pv: %w", err) - } - } - for _, pvc := range []*v1.PersistentVolumeClaim{pvc1, pvc2} { - _, err := testutils.CreatePVC(cs, pvc) - if err != nil { - return fmt.Errorf("cannot create pvc: %w", err) - } - } - return nil - }, - existingPods: []*v1.Pod{ - initPausePod(&testutils.PausePodConfig{ - Name: "victim-pod-1", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Volumes: []v1.Volume{{ - Name: "volume", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc1.Name, - }, - }, - }}, - }), - initPausePod(&testutils.PausePodConfig{ - Name: "victim-pod-2", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Volumes: []v1.Volume{{ - Name: "volume", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc2.Name, - }, - }, - }}, - }), - }, - pod: initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Volumes: []v1.Volume{ - { - Name: "volume-1", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc1.Name, - }, - }, - }, - { - Name: "volume-2", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc2.Name, - }, - }, - }, - }, - }), - preemptedPodIndexes: map[int]struct{}{0: {}, 1: {}}, - cleanup: func() error { - for _, pvc := range []*v1.PersistentVolumeClaim{pvc1, pvc2} { - if err := testutils.DeletePVC(cs, pvc.Name, pvc.Namespace); err != nil { - return fmt.Errorf("cannot delete pvc: %w", err) - } - } - for _, pv := range []*v1.PersistentVolume{pv1, pv2} { - if err := testutils.DeletePV(cs, pv.Name); err != nil { - return fmt.Errorf("cannot delete pv: %w", err) - } - } - return nil - }, - }, - { - name: "preempt single pod with two volumes", - init: func() error { - for _, pv := range []*v1.PersistentVolume{pv1, pv2} { - _, err := testutils.CreatePV(cs, pv) - if err != nil { - return fmt.Errorf("cannot create pv: %w", err) - } - } - for _, pvc := range []*v1.PersistentVolumeClaim{pvc1, pvc2} { - _, err := testutils.CreatePVC(cs, pvc) - if err != nil { - return fmt.Errorf("cannot create pvc: %w", err) - } - } - return nil - }, - existingPods: []*v1.Pod{ - initPausePod(&testutils.PausePodConfig{ - Name: "victim-pod", - Namespace: testCtx.NS.Name, - Priority: &lowPriority, - Volumes: []v1.Volume{ - { - Name: "volume-1", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc1.Name, - }, - }, - }, - { - Name: "volume-2", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc2.Name, - }, - }, - }, - }, - }), - }, - pod: initPausePod(&testutils.PausePodConfig{ - Name: "preemptor-pod", - Namespace: testCtx.NS.Name, - Priority: &highPriority, - Volumes: []v1.Volume{ - { - Name: "volume-1", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc1.Name, - }, - }, - }, - { - Name: "volume-2", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc2.Name, - }, - }, - }, - }, - }), - preemptedPodIndexes: map[int]struct{}{0: {}}, - cleanup: func() error { - for _, pvc := range []*v1.PersistentVolumeClaim{pvc1, pvc2} { - if err := testutils.DeletePVC(cs, pvc.Name, pvc.Namespace); err != nil { - return fmt.Errorf("cannot delete pvc: %w", err) - } - } - for _, pv := range []*v1.PersistentVolume{pv1, pv2} { - if err := testutils.DeletePV(cs, pv.Name); err != nil { - return fmt.Errorf("cannot delete pv: %w", err) - } - } - return nil - }, - }, - } - - // Create a node with some resources and a label. - nodeRes := map[v1.ResourceName]string{ - v1.ResourcePods: "32", - v1.ResourceCPU: "500m", - v1.ResourceMemory: "500", - } - nodeObject := st.MakeNode().Name("node1").Capacity(nodeRes).Label("node", "node1").Obj() - if _, err := createNode(cs, nodeObject); err != nil { - t.Fatalf("Error creating node: %v", err) - } - - for _, asyncPreemptionEnabled := range []bool{true, false} { - for _, clearingNominatedNodeNameAfterBinding := range []bool{true, false} { - for _, test := range tests { - t.Run(fmt.Sprintf("%s (Async preemption enabled: %v, ClearingNominatedNodeNameAfterBinding: %v)", test.name, asyncPreemptionEnabled, clearingNominatedNodeNameAfterBinding), func(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ClearingNominatedNodeNameAfterBinding, clearingNominatedNodeNameAfterBinding) - - if err := test.init(); err != nil { - t.Fatalf("Error while initializing test: %v", err) - } - - pods := make([]*v1.Pod, len(test.existingPods)) - t.Cleanup(func() { - testutils.CleanupPods(testCtx.Ctx, cs, t, pods) - if err := test.cleanup(); err != nil { - t.Errorf("Error cleaning up test: %v", err) - } - }) - // Create and run existingPods. - for i, p := range test.existingPods { - var err error - pods[i], err = runPausePod(cs, p) - if err != nil { - t.Fatalf("Error running pause pod: %v", err) - } - } - // Create the "pod". - preemptor, err := createPausePod(cs, test.pod) - if err != nil { - t.Errorf("Error while creating high priority pod: %v", err) - } - pods = append(pods, preemptor) - // Wait for preemption of pods and make sure the other ones are not preempted. - for i, p := range pods { - if _, found := test.preemptedPodIndexes[i]; found { - if err = wait.PollUntilContextTimeout(testCtx.Ctx, time.Second, wait.ForeverTestTimeout, false, - podIsGettingEvicted(cs, p.Namespace, p.Name)); err != nil { - t.Errorf("Pod %v/%v is not getting evicted.", p.Namespace, p.Name) - } - } else { - if p.DeletionTimestamp != nil { - t.Errorf("Didn't expect pod %v to get preempted.", p.Name) - } - } - } - // Also check that the preemptor pod gets the NominatedNodeName field set. - if len(test.preemptedPodIndexes) > 0 && !clearingNominatedNodeNameAfterBinding { - if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil { - t.Errorf("NominatedNodeName field was not set for pod %v: %v", preemptor.Name, err) - } - } - }) - } - } - } -} diff --git a/test/integration/scheduler/preemption/nominatednodename/nominatednodename_test.go b/test/integration/scheduler/preemption/nominatednodename/nominatednodename_test.go index 38cf9706c6069..00fcbbf83810f 100644 --- a/test/integration/scheduler/preemption/nominatednodename/nominatednodename_test.go +++ b/test/integration/scheduler/preemption/nominatednodename/nominatednodename_test.go @@ -119,10 +119,10 @@ func TestNominatedNode(t *testing.T) { st.MakePod().Name("low-4").Priority(lowPriority).Req(map[v1.ResourceName]string{v1.ResourceCPU: "1"}).Obj(), }, { - st.MakePod().Name("medium").Priority(mediumPriority).Req(map[v1.ResourceName]string{v1.ResourceCPU: "4"}).Obj(), + st.MakePod().Name("medium").Priority(mediumPriority).Req(map[v1.ResourceName]string{v1.ResourceCPU: "3"}).Obj(), }, { - st.MakePod().Name("high").Priority(highPriority).Req(map[v1.ResourceName]string{v1.ResourceCPU: "3"}).Obj(), + st.MakePod().Name("high").Priority(highPriority).Req(map[v1.ResourceName]string{v1.ResourceCPU: "4"}).Obj(), }, }, postChecks: []func(ctx context.Context, cs clientset.Interface, pod *v1.Pod) error{ @@ -130,6 +130,7 @@ func TestNominatedNode(t *testing.T) { testutils.WaitForNominatedNodeName, testutils.WaitForNominatedNodeName, }, + podNamesToDelete: []string{"low-1", "low-2", "low-3", "low-4"}, expectNilNominatedNodeName: true, }, { diff --git a/test/integration/scheduler/preemption/preemption_test.go b/test/integration/scheduler/preemption/preemption_test.go index 18d71c492418e..4c1535c148e45 100644 --- a/test/integration/scheduler/preemption/preemption_test.go +++ b/test/integration/scheduler/preemption/preemption_test.go @@ -25,16 +25,24 @@ import ( "time" v1 "k8s.io/api/core/v1" + policy "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/component-helpers/storage/volume" "k8s.io/klog/v2" configv1 "k8s.io/kube-scheduler/config/v1" fwk "k8s.io/kube-scheduler/framework" podutil "k8s.io/kubernetes/pkg/api/v1/pod" + "k8s.io/kubernetes/pkg/apis/scheduling" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler" "k8s.io/kubernetes/pkg/scheduler/apis/config" @@ -44,9 +52,11 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultpreemption" plfeature "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumerestrictions" "k8s.io/kubernetes/pkg/scheduler/framework/preemption" frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" st "k8s.io/kubernetes/pkg/scheduler/testing" + "k8s.io/kubernetes/plugin/pkg/admission/priority" testutils "k8s.io/kubernetes/test/integration/util" "k8s.io/kubernetes/test/utils/ktesting" "k8s.io/utils/ptr" @@ -54,11 +64,19 @@ import ( // imported from testutils var ( - initPausePod = testutils.InitPausePod - createNode = testutils.CreateNode - createPausePod = testutils.CreatePausePod - runPausePod = testutils.RunPausePod - podIsGettingEvicted = testutils.PodIsGettingEvicted + initPausePod = testutils.InitPausePod + createNode = testutils.CreateNode + createPausePod = testutils.CreatePausePod + runPausePod = testutils.RunPausePod + initTest = testutils.InitTestSchedulerWithNS + initTestDisablePreemption = testutils.InitTestDisablePreemption + initDisruptionController = testutils.InitDisruptionController + waitCachedPodsStable = testutils.WaitCachedPodsStable + podIsGettingEvicted = testutils.PodIsGettingEvicted + podUnschedulable = testutils.PodUnschedulable + waitForPDBsStable = testutils.WaitForPDBsStable + waitForPodToScheduleWithTimeout = testutils.WaitForPodToScheduleWithTimeout + waitForPodUnschedulable = testutils.WaitForPodUnschedulable ) const filterPluginName = "filter-plugin" @@ -444,7 +462,7 @@ func TestPreemption(t *testing.T) { // Wait for preemption of pods and make sure the other ones are not preempted. for i, p := range pods { if _, found := test.preemptedPodIndexes[i]; found { - if err = wait.PollUntilContextTimeout(testCtx.Ctx, 200*time.Millisecond, wait.ForeverTestTimeout, false, + if err = wait.PollUntilContextTimeout(testCtx.Ctx, time.Second, wait.ForeverTestTimeout, false, podIsGettingEvicted(cs, p.Namespace, p.Name)); err != nil { t.Errorf("Pod %v/%v is not getting evicted.", p.Namespace, p.Name) } @@ -988,3 +1006,1173 @@ func podInUnschedulablePodPool(t *testing.T, queue queue.SchedulingQueue, podNam } return false } + +// TestNonPreemption tests NonPreempt option of PriorityClass of scheduler works as expected. +func TestNonPreemption(t *testing.T) { + var preemptNever = v1.PreemptNever + // Initialize scheduler. + testCtx := initTest(t, "non-preemption") + cs := testCtx.ClientSet + tests := []struct { + name string + PreemptionPolicy *v1.PreemptionPolicy + }{ + { + name: "pod preemption will happen", + PreemptionPolicy: nil, + }, + { + name: "pod preemption will not happen", + PreemptionPolicy: &preemptNever, + }, + } + victim := initPausePod(&testutils.PausePodConfig{ + Name: "victim-pod", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(400, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, + }, + }) + + preemptor := initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(300, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, + }, + }) + + // Create a node with some resources + nodeRes := map[v1.ResourceName]string{ + v1.ResourcePods: "32", + v1.ResourceCPU: "500m", + v1.ResourceMemory: "500", + } + _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) + if err != nil { + t.Fatalf("Error creating nodes: %v", err) + } + + for _, asyncPreemptionEnabled := range []bool{true, false} { + for _, test := range tests { + t.Run(fmt.Sprintf("%s (Async preemption enabled: %v)", test.name, asyncPreemptionEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) + + defer testutils.CleanupPods(testCtx.Ctx, cs, t, []*v1.Pod{preemptor, victim}) + preemptor.Spec.PreemptionPolicy = test.PreemptionPolicy + victimPod, err := createPausePod(cs, victim) + if err != nil { + t.Fatalf("Error while creating victim: %v", err) + } + if err := waitForPodToScheduleWithTimeout(testCtx.Ctx, cs, victimPod, 5*time.Second); err != nil { + t.Fatalf("victim %v should be become scheduled", victimPod.Name) + } + + preemptorPod, err := createPausePod(cs, preemptor) + if err != nil { + t.Fatalf("Error while creating preemptor: %v", err) + } + + err = testutils.WaitForNominatedNodeNameWithTimeout(testCtx.Ctx, cs, preemptorPod, 5*time.Second) + // test.PreemptionPolicy == nil means we expect the preemptor to be nominated. + expect := test.PreemptionPolicy == nil + // err == nil indicates the preemptor is indeed nominated. + got := err == nil + if got != expect { + t.Errorf("Expect preemptor to be nominated=%v, but got=%v", expect, got) + } + }) + } + } +} + +// TestDisablePreemption tests disable pod preemption of scheduler works as expected. +func TestDisablePreemption(t *testing.T) { + // Initialize scheduler, and disable preemption. + testCtx := initTestDisablePreemption(t, "disable-preemption") + cs := testCtx.ClientSet + + tests := []struct { + name string + existingPods []*v1.Pod + pod *v1.Pod + }{ + { + name: "pod preemption will not happen", + existingPods: []*v1.Pod{ + initPausePod(&testutils.PausePodConfig{ + Name: "victim-pod", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(400, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, + }, + }), + }, + pod: initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(300, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, + }, + }), + }, + } + + // Create a node with some resources + nodeRes := map[v1.ResourceName]string{ + v1.ResourcePods: "32", + v1.ResourceCPU: "500m", + v1.ResourceMemory: "500", + } + _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) + if err != nil { + t.Fatalf("Error creating nodes: %v", err) + } + + for _, asyncPreemptionEnabled := range []bool{true, false} { + for _, test := range tests { + t.Run(fmt.Sprintf("%s (Async preemption enabled: %v)", test.name, asyncPreemptionEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) + + pods := make([]*v1.Pod, len(test.existingPods)) + // Create and run existingPods. + for i, p := range test.existingPods { + pods[i], err = runPausePod(cs, p) + if err != nil { + t.Fatalf("Test [%v]: Error running pause pod: %v", test.name, err) + } + } + // Create the "pod". + preemptor, err := createPausePod(cs, test.pod) + if err != nil { + t.Errorf("Error while creating high priority pod: %v", err) + } + // Ensure preemptor should keep unschedulable. + if err := waitForPodUnschedulable(testCtx.Ctx, cs, preemptor); err != nil { + t.Errorf("Preemptor %v should not become scheduled", preemptor.Name) + } + + // Ensure preemptor should not be nominated. + if err := testutils.WaitForNominatedNodeNameWithTimeout(testCtx.Ctx, cs, preemptor, 5*time.Second); err == nil { + t.Errorf("Preemptor %v should not be nominated", preemptor.Name) + } + + // Cleanup + pods = append(pods, preemptor) + testutils.CleanupPods(testCtx.Ctx, cs, t, pods) + }) + } + } +} + +// This test verifies that system critical priorities are created automatically and resolved properly. +func TestPodPriorityResolution(t *testing.T) { + admission := priority.NewPlugin() + testCtx := testutils.InitTestScheduler(t, testutils.InitTestAPIServer(t, "preemption", admission)) + cs := testCtx.ClientSet + + // Build clientset and informers for controllers. + externalClientConfig := restclient.CopyConfig(testCtx.KubeConfig) + externalClientConfig.QPS = -1 + externalClientset := clientset.NewForConfigOrDie(externalClientConfig) + externalInformers := informers.NewSharedInformerFactory(externalClientset, time.Second) + admission.SetExternalKubeClientSet(externalClientset) + admission.SetExternalKubeInformerFactory(externalInformers) + + // Waiting for all controllers to sync + testutils.SyncSchedulerInformerFactory(testCtx) + externalInformers.Start(testCtx.Ctx.Done()) + externalInformers.WaitForCacheSync(testCtx.Ctx.Done()) + + // Run all controllers + go testCtx.Scheduler.Run(testCtx.Ctx) + + tests := []struct { + Name string + PriorityClass string + Pod *v1.Pod + ExpectedPriority int32 + ExpectedError error + }{ + { + Name: "SystemNodeCritical priority class", + PriorityClass: scheduling.SystemNodeCritical, + ExpectedPriority: scheduling.SystemCriticalPriority + 1000, + Pod: initPausePod(&testutils.PausePodConfig{ + Name: fmt.Sprintf("pod1-%v", scheduling.SystemNodeCritical), + Namespace: metav1.NamespaceSystem, + PriorityClassName: scheduling.SystemNodeCritical, + }), + }, + { + Name: "SystemClusterCritical priority class", + PriorityClass: scheduling.SystemClusterCritical, + ExpectedPriority: scheduling.SystemCriticalPriority, + Pod: initPausePod(&testutils.PausePodConfig{ + Name: fmt.Sprintf("pod2-%v", scheduling.SystemClusterCritical), + Namespace: metav1.NamespaceSystem, + PriorityClassName: scheduling.SystemClusterCritical, + }), + }, + { + Name: "Invalid priority class should result in error", + PriorityClass: "foo", + ExpectedPriority: scheduling.SystemCriticalPriority, + Pod: initPausePod(&testutils.PausePodConfig{ + Name: fmt.Sprintf("pod3-%v", scheduling.SystemClusterCritical), + Namespace: metav1.NamespaceSystem, + PriorityClassName: "foo", + }), + ExpectedError: fmt.Errorf("failed to create pause pod: pods \"pod3-system-cluster-critical\" is forbidden: no PriorityClass with name foo was found"), + }, + } + + // Create a node with some resources + nodeRes := map[v1.ResourceName]string{ + v1.ResourcePods: "32", + v1.ResourceCPU: "500m", + v1.ResourceMemory: "500", + } + _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) + if err != nil { + t.Fatalf("Error creating nodes: %v", err) + } + + pods := make([]*v1.Pod, 0, len(tests)) + for _, asyncPreemptionEnabled := range []bool{true, false} { + for _, test := range tests { + t.Run(fmt.Sprintf("%s (Async preemption enabled: %v)", test.Name, asyncPreemptionEnabled), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) + + pod, err := runPausePod(cs, test.Pod) + if err != nil { + if test.ExpectedError == nil { + t.Fatalf("Test [PodPriority/%v]: Error running pause pod: %v", test.PriorityClass, err) + } + if err.Error() != test.ExpectedError.Error() { + t.Fatalf("Test [PodPriority/%v]: Expected error %v but got error %v", test.PriorityClass, test.ExpectedError, err) + } + return + } + pods = append(pods, pod) + if pod.Spec.Priority != nil { + if *pod.Spec.Priority != test.ExpectedPriority { + t.Errorf("Expected pod %v to have priority %v but was %v", pod.Name, test.ExpectedPriority, pod.Spec.Priority) + } + } else { + t.Errorf("Expected pod %v to have priority %v but was nil", pod.Name, test.PriorityClass) + } + testutils.CleanupPods(testCtx.Ctx, cs, t, pods) + }) + } + } + testutils.CleanupNodes(cs, t) +} + +func mkPriorityPodWithGrace(tc *testutils.TestContext, name string, priority int32, grace int64) *v1.Pod { + defaultPodRes := &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(100, resource.DecimalSI)}, + } + pod := initPausePod(&testutils.PausePodConfig{ + Name: name, + Namespace: tc.NS.Name, + Priority: &priority, + Labels: map[string]string{"pod": name}, + Resources: defaultPodRes, + }) + pod.Spec.TerminationGracePeriodSeconds = &grace + return pod +} + +// This test ensures that while the preempting pod is waiting for the victims to +// terminate, other pending lower priority pods are not scheduled in the room created +// after preemption and while the higher priority pods is not scheduled yet. +func TestPreemptionStarvation(t *testing.T) { + // Initialize scheduler. + testCtx := initTest(t, "preemption") + cs := testCtx.ClientSet + + tests := []struct { + name string + numExistingPod int + numExpectedPending int + preemptor *v1.Pod + }{ + { + // This test ensures that while the preempting pod is waiting for the victims + // terminate, other lower priority pods are not scheduled in the room created + // after preemption and while the higher priority pods is not scheduled yet. + name: "starvation test: higher priority pod is scheduled before the lower priority ones", + numExistingPod: 10, + numExpectedPending: 5, + preemptor: initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(300, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, + }, + }), + }, + } + + // Create a node with some resources + nodeRes := map[v1.ResourceName]string{ + v1.ResourcePods: "32", + v1.ResourceCPU: "500m", + v1.ResourceMemory: "500", + } + _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) + if err != nil { + t.Fatalf("Error creating nodes: %v", err) + } + + for _, asyncPreemptionEnabled := range []bool{true, false} { + for _, clearingNominatedNodeNameAfterBinding := range []bool{true, false} { + for _, test := range tests { + t.Run(fmt.Sprintf("%s (Async preemption enabled: %v, ClearingNominatedNodeNameAfterBinding: %v)", test.name, asyncPreemptionEnabled, clearingNominatedNodeNameAfterBinding), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ClearingNominatedNodeNameAfterBinding, clearingNominatedNodeNameAfterBinding) + + pendingPods := make([]*v1.Pod, test.numExpectedPending) + numRunningPods := test.numExistingPod - test.numExpectedPending + runningPods := make([]*v1.Pod, numRunningPods) + // Create and run existingPods. + for i := 0; i < numRunningPods; i++ { + runningPods[i], err = createPausePod(cs, mkPriorityPodWithGrace(testCtx, fmt.Sprintf("rpod-%v", i), mediumPriority, 0)) + if err != nil { + t.Fatalf("Error creating pause pod: %v", err) + } + } + // make sure that runningPods are all scheduled. + for _, p := range runningPods { + if err := testutils.WaitForPodToSchedule(testCtx.Ctx, cs, p); err != nil { + t.Fatalf("Pod %v/%v didn't get scheduled: %v", p.Namespace, p.Name, err) + } + } + // Create pending pods. + for i := 0; i < test.numExpectedPending; i++ { + pendingPods[i], err = createPausePod(cs, mkPriorityPodWithGrace(testCtx, fmt.Sprintf("ppod-%v", i), mediumPriority, 0)) + if err != nil { + t.Fatalf("Error creating pending pod: %v", err) + } + } + // Make sure that all pending pods are being marked unschedulable. + for _, p := range pendingPods { + if err := wait.PollUntilContextTimeout(testCtx.Ctx, 100*time.Millisecond, wait.ForeverTestTimeout, false, + podUnschedulable(cs, p.Namespace, p.Name)); err != nil { + t.Errorf("Pod %v/%v didn't get marked unschedulable: %v", p.Namespace, p.Name, err) + } + } + // Create the preemptor. + preemptor, err := createPausePod(cs, test.preemptor) + if err != nil { + t.Errorf("Error while creating the preempting pod: %v", err) + } + + // Make sure that preemptor is scheduled after preemptions. + if err := testutils.WaitForPodToScheduleWithTimeout(testCtx.Ctx, cs, preemptor, 60*time.Second); err != nil { + t.Errorf("Preemptor pod %v didn't get scheduled: %v", preemptor.Name, err) + } + + // Check if .status.nominatedNodeName of the preemptor pod gets set when feature gate is disabled. + // This test always expects preemption to occur since numExistingPod (10) fills the node completely. + if !clearingNominatedNodeNameAfterBinding { + if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil { + t.Errorf(".status.nominatedNodeName was not set for pod %v/%v: %v", preemptor.Namespace, preemptor.Name, err) + } + } + // Cleanup + klog.Info("Cleaning up all pods...") + allPods := pendingPods + allPods = append(allPods, runningPods...) + allPods = append(allPods, preemptor) + testutils.CleanupPods(testCtx.Ctx, cs, t, allPods) + }) + } + } + } +} + +// TestPreemptionRaces tests that other scheduling events and operations do not +// race with the preemption process. +func TestPreemptionRaces(t *testing.T) { + // Initialize scheduler. + testCtx := initTest(t, "preemption-race") + cs := testCtx.ClientSet + + tests := []struct { + name string + numInitialPods int // Pods created and executed before running preemptor + numAdditionalPods int // Pods created after creating the preemptor + numRepetitions int // Repeat the tests to check races + preemptor *v1.Pod + }{ + { + // This test ensures that while the preempting pod is waiting for the victims + // terminate, other lower priority pods are not scheduled in the room created + // after preemption and while the higher priority pods is not scheduled yet. + name: "ensures that other pods are not scheduled while preemptor is being marked as nominated (issue #72124)", + numInitialPods: 2, + numAdditionalPods: 20, + numRepetitions: 5, + preemptor: initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(4900, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(4900, resource.DecimalSI)}, + }, + }), + }, + } + + // Create a node with some resources + nodeRes := map[v1.ResourceName]string{ + v1.ResourcePods: "100", + v1.ResourceCPU: "5000m", + v1.ResourceMemory: "5000", + } + _, err := createNode(testCtx.ClientSet, st.MakeNode().Name("node1").Capacity(nodeRes).Obj()) + if err != nil { + t.Fatalf("Error creating nodes: %v", err) + } + + for _, asyncPreemptionEnabled := range []bool{true, false} { + for _, clearingNominatedNodeNameAfterBinding := range []bool{true, false} { + for _, test := range tests { + t.Run(fmt.Sprintf("%s (Async preemption enabled: %v, ClearingNominatedNodeNameAfterBinding: %v)", test.name, asyncPreemptionEnabled, clearingNominatedNodeNameAfterBinding), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ClearingNominatedNodeNameAfterBinding, clearingNominatedNodeNameAfterBinding) + + if test.numRepetitions <= 0 { + test.numRepetitions = 1 + } + for n := 0; n < test.numRepetitions; n++ { + initialPods := make([]*v1.Pod, test.numInitialPods) + additionalPods := make([]*v1.Pod, test.numAdditionalPods) + // Create and run existingPods. + for i := 0; i < test.numInitialPods; i++ { + initialPods[i], err = createPausePod(cs, mkPriorityPodWithGrace(testCtx, fmt.Sprintf("rpod-%v", i), mediumPriority, 0)) + if err != nil { + t.Fatalf("Error creating pause pod: %v", err) + } + } + // make sure that initial Pods are all scheduled. + for _, p := range initialPods { + if err := testutils.WaitForPodToSchedule(testCtx.Ctx, cs, p); err != nil { + t.Fatalf("Pod %v/%v didn't get scheduled: %v", p.Namespace, p.Name, err) + } + } + // Create the preemptor. + klog.Info("Creating the preemptor pod...") + preemptor, err := createPausePod(cs, test.preemptor) + if err != nil { + t.Errorf("Error while creating the preempting pod: %v", err) + } + + klog.Info("Creating additional pods...") + for i := 0; i < test.numAdditionalPods; i++ { + additionalPods[i], err = createPausePod(cs, mkPriorityPodWithGrace(testCtx, fmt.Sprintf("ppod-%v", i), mediumPriority, 0)) + if err != nil { + t.Fatalf("Error creating pending pod: %v", err) + } + } + // Make sure that preemptor is scheduled after preemptions. + if err := testutils.WaitForPodToScheduleWithTimeout(testCtx.Ctx, cs, preemptor, 60*time.Second); err != nil { + t.Errorf("Preemptor pod %v didn't get scheduled: %v", preemptor.Name, err) + } + + // Check that the preemptor pod gets nominated node name when feature gate is disabled. + if !clearingNominatedNodeNameAfterBinding { + if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil { + t.Errorf(".status.nominatedNodeName was not set for pod %v/%v: %v", preemptor.Namespace, preemptor.Name, err) + } + } + + klog.Info("Check unschedulable pods still exists and were never scheduled...") + for _, p := range additionalPods { + pod, err := cs.CoreV1().Pods(p.Namespace).Get(testCtx.Ctx, p.Name, metav1.GetOptions{}) + if err != nil { + t.Errorf("Error in getting Pod %v/%v info: %v", p.Namespace, p.Name, err) + } + if len(pod.Spec.NodeName) > 0 { + t.Errorf("Pod %v/%v is already scheduled", p.Namespace, p.Name) + } + _, cond := podutil.GetPodCondition(&pod.Status, v1.PodScheduled) + if cond != nil && cond.Status != v1.ConditionFalse { + t.Errorf("Pod %v/%v is no longer unschedulable: %v", p.Namespace, p.Name, err) + } + } + // Cleanup + klog.Info("Cleaning up all pods...") + allPods := additionalPods + allPods = append(allPods, initialPods...) + allPods = append(allPods, preemptor) + testutils.CleanupPods(testCtx.Ctx, cs, t, allPods) + } + }) + } + } + } +} + +func mkMinAvailablePDB(name, namespace string, uid types.UID, minAvailable int, matchLabels map[string]string) *policy.PodDisruptionBudget { + intMinAvailable := intstr.FromInt32(int32(minAvailable)) + return &policy.PodDisruptionBudget{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: policy.PodDisruptionBudgetSpec{ + MinAvailable: &intMinAvailable, + Selector: &metav1.LabelSelector{MatchLabels: matchLabels}, + }, + } +} + +func addPodConditionReady(pod *v1.Pod) { + pod.Status = v1.PodStatus{ + Phase: v1.PodRunning, + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + }, + } +} + +// TestPDBInPreemption tests PodDisruptionBudget support in preemption. +func TestPDBInPreemption(t *testing.T) { + // Initialize scheduler. + testCtx := initTest(t, "preemption-pdb") + cs := testCtx.ClientSet + + initDisruptionController(t, testCtx) + + defaultPodRes := &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(100, resource.DecimalSI)}, + } + defaultNodeRes := map[v1.ResourceName]string{ + v1.ResourcePods: "32", + v1.ResourceCPU: "500m", + v1.ResourceMemory: "500", + } + + tests := []struct { + name string + nodeCnt int + pdbs []*policy.PodDisruptionBudget + pdbPodNum []int32 + existingPods []*v1.Pod + pod *v1.Pod + preemptedPodIndexes map[int]struct{} + }{ + { + name: "A non-PDB violating pod is preempted despite its higher priority", + nodeCnt: 1, + pdbs: []*policy.PodDisruptionBudget{ + mkMinAvailablePDB("pdb-1", testCtx.NS.Name, types.UID("pdb-1-uid"), 2, map[string]string{"foo": "bar"}), + }, + pdbPodNum: []int32{2}, + existingPods: []*v1.Pod{ + initPausePod(&testutils.PausePodConfig{ + Name: "low-pod1", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: defaultPodRes, + Labels: map[string]string{"foo": "bar"}, + }), + initPausePod(&testutils.PausePodConfig{ + Name: "low-pod2", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: defaultPodRes, + Labels: map[string]string{"foo": "bar"}, + }), + initPausePod(&testutils.PausePodConfig{ + Name: "mid-pod3", + Namespace: testCtx.NS.Name, + Priority: &mediumPriority, + Resources: defaultPodRes, + }), + }, + pod: initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(300, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, + }, + }), + preemptedPodIndexes: map[int]struct{}{2: {}}, + }, + { + name: "A node without any PDB violating pods is preferred for preemption", + nodeCnt: 2, + pdbs: []*policy.PodDisruptionBudget{ + mkMinAvailablePDB("pdb-1", testCtx.NS.Name, types.UID("pdb-1-uid"), 2, map[string]string{"foo": "bar"}), + }, + pdbPodNum: []int32{1}, + existingPods: []*v1.Pod{ + initPausePod(&testutils.PausePodConfig{ + Name: "low-pod1", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: defaultPodRes, + NodeName: "node-1", + Labels: map[string]string{"foo": "bar"}, + }), + initPausePod(&testutils.PausePodConfig{ + Name: "mid-pod2", + Namespace: testCtx.NS.Name, + Priority: &mediumPriority, + NodeName: "node-2", + Resources: defaultPodRes, + }), + }, + pod: initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, + }, + }), + preemptedPodIndexes: map[int]struct{}{1: {}}, + }, + { + name: "A node with fewer PDB violating pods is preferred for preemption", + nodeCnt: 3, + pdbs: []*policy.PodDisruptionBudget{ + mkMinAvailablePDB("pdb-1", testCtx.NS.Name, types.UID("pdb-1-uid"), 2, map[string]string{"foo1": "bar"}), + mkMinAvailablePDB("pdb-2", testCtx.NS.Name, types.UID("pdb-2-uid"), 2, map[string]string{"foo2": "bar"}), + }, + pdbPodNum: []int32{1, 5}, + existingPods: []*v1.Pod{ + initPausePod(&testutils.PausePodConfig{ + Name: "low-pod1", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: defaultPodRes, + NodeName: "node-1", + Labels: map[string]string{"foo1": "bar"}, + }), + initPausePod(&testutils.PausePodConfig{ + Name: "mid-pod1", + Namespace: testCtx.NS.Name, + Priority: &mediumPriority, + Resources: defaultPodRes, + NodeName: "node-1", + }), + initPausePod(&testutils.PausePodConfig{ + Name: "low-pod2", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: defaultPodRes, + NodeName: "node-2", + Labels: map[string]string{"foo2": "bar"}, + }), + initPausePod(&testutils.PausePodConfig{ + Name: "mid-pod2", + Namespace: testCtx.NS.Name, + Priority: &mediumPriority, + Resources: defaultPodRes, + NodeName: "node-2", + Labels: map[string]string{"foo2": "bar"}, + }), + initPausePod(&testutils.PausePodConfig{ + Name: "low-pod4", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: defaultPodRes, + NodeName: "node-3", + Labels: map[string]string{"foo2": "bar"}, + }), + initPausePod(&testutils.PausePodConfig{ + Name: "low-pod5", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: defaultPodRes, + NodeName: "node-3", + Labels: map[string]string{"foo2": "bar"}, + }), + initPausePod(&testutils.PausePodConfig{ + Name: "low-pod6", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Resources: defaultPodRes, + NodeName: "node-3", + Labels: map[string]string{"foo2": "bar"}, + }), + }, + pod: initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Resources: &v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(500, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(400, resource.DecimalSI)}, + }, + }), + // The third node is chosen because PDB is not violated for node 3 and the victims have lower priority than node-2. + preemptedPodIndexes: map[int]struct{}{4: {}, 5: {}, 6: {}}, + }, + } + + for _, asyncPreemptionEnabled := range []bool{true, false} { + for _, clearingNominatedNodeNameAfterBinding := range []bool{true, false} { + for _, test := range tests { + t.Run(fmt.Sprintf("%s (Async preemption enabled: %v, ClearingNominatedNodeNameAfterBinding: %v)", test.name, asyncPreemptionEnabled, clearingNominatedNodeNameAfterBinding), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ClearingNominatedNodeNameAfterBinding, clearingNominatedNodeNameAfterBinding) + + for i := 1; i <= test.nodeCnt; i++ { + nodeName := fmt.Sprintf("node-%v", i) + _, err := createNode(cs, st.MakeNode().Name(nodeName).Capacity(defaultNodeRes).Obj()) + if err != nil { + t.Fatalf("Error creating node %v: %v", nodeName, err) + } + } + + pods := make([]*v1.Pod, len(test.existingPods)) + var err error + // Create and run existingPods. + for i, p := range test.existingPods { + if pods[i], err = runPausePod(cs, p); err != nil { + t.Fatalf("Test [%v]: Error running pause pod: %v", test.name, err) + } + // Add pod condition ready so that PDB is updated. + addPodConditionReady(p) + if _, err := testCtx.ClientSet.CoreV1().Pods(testCtx.NS.Name).UpdateStatus(testCtx.Ctx, p, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } + // Wait for Pods to be stable in scheduler cache. + if err := waitCachedPodsStable(testCtx, test.existingPods); err != nil { + t.Fatalf("Not all pods are stable in the cache: %v", err) + } + + // Create PDBs. + for _, pdb := range test.pdbs { + _, err := testCtx.ClientSet.PolicyV1().PodDisruptionBudgets(testCtx.NS.Name).Create(testCtx.Ctx, pdb, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create PDB: %v", err) + } + } + // Wait for PDBs to become stable. + if err := waitForPDBsStable(testCtx, test.pdbs, test.pdbPodNum); err != nil { + t.Fatalf("Not all pdbs are stable in the cache: %v", err) + } + + // Create the "pod". + preemptor, err := createPausePod(cs, test.pod) + if err != nil { + t.Errorf("Error while creating high priority pod: %v", err) + } + // Wait for preemption of pods and make sure the other ones are not preempted. + for i, p := range pods { + if _, found := test.preemptedPodIndexes[i]; found { + if err = wait.PollUntilContextTimeout(testCtx.Ctx, time.Second, wait.ForeverTestTimeout, false, + podIsGettingEvicted(cs, p.Namespace, p.Name)); err != nil { + t.Errorf("Test [%v]: Pod %v/%v is not getting evicted.", test.name, p.Namespace, p.Name) + } + } else { + if p.DeletionTimestamp != nil { + t.Errorf("Test [%v]: Didn't expect pod %v/%v to get preempted.", test.name, p.Namespace, p.Name) + } + } + } + // Also check if .status.nominatedNodeName of the preemptor pod gets set. + if len(test.preemptedPodIndexes) > 0 && !clearingNominatedNodeNameAfterBinding { + if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil { + t.Errorf("Test [%v]: .status.nominatedNodeName was not set for pod %v/%v: %v", test.name, preemptor.Namespace, preemptor.Name, err) + } + } + + // Cleanup + pods = append(pods, preemptor) + testutils.CleanupPods(testCtx.Ctx, cs, t, pods) + if err := cs.PolicyV1().PodDisruptionBudgets(testCtx.NS.Name).DeleteCollection(testCtx.Ctx, metav1.DeleteOptions{}, metav1.ListOptions{}); err != nil { + t.Errorf("error while deleting PDBs, error: %v", err) + } + if err := cs.CoreV1().Nodes().DeleteCollection(testCtx.Ctx, metav1.DeleteOptions{}, metav1.ListOptions{}); err != nil { + t.Errorf("error whiling deleting nodes, error: %v", err) + } + }) + } + } + } +} + +// TestReadWriteOncePodPreemption tests preemption scenarios for pods with +// ReadWriteOncePod PVCs. +func TestReadWriteOncePodPreemption(t *testing.T) { + cfg := configtesting.V1ToInternalWithDefaults(t, configv1.KubeSchedulerConfiguration{ + Profiles: []configv1.KubeSchedulerProfile{{ + SchedulerName: ptr.To(v1.DefaultSchedulerName), + Plugins: &configv1.Plugins{ + Filter: configv1.PluginSet{ + Enabled: []configv1.Plugin{ + {Name: volumerestrictions.Name}, + }, + }, + PreFilter: configv1.PluginSet{ + Enabled: []configv1.Plugin{ + {Name: volumerestrictions.Name}, + }, + }, + }, + }}, + }) + + testCtx := testutils.InitTestSchedulerWithOptions(t, + testutils.InitTestAPIServer(t, "preemption", nil), + 0, + scheduler.WithProfiles(cfg.Profiles...)) + testutils.SyncSchedulerInformerFactory(testCtx) + go testCtx.Scheduler.Run(testCtx.Ctx) + + cs := testCtx.ClientSet + + storage := v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}} + volType := v1.HostPathDirectoryOrCreate + pv1 := st.MakePersistentVolume(). + Name("pv-with-read-write-once-pod-1"). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). + Capacity(storage.Requests). + HostPathVolumeSource(&v1.HostPathVolumeSource{Path: "/mnt1", Type: &volType}). + Obj() + pvc1 := st.MakePersistentVolumeClaim(). + Name("pvc-with-read-write-once-pod-1"). + Namespace(testCtx.NS.Name). + // Annotation and volume name required for PVC to be considered bound. + Annotation(volume.AnnBindCompleted, "true"). + VolumeName(pv1.Name). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). + Resources(storage). + Obj() + pv2 := st.MakePersistentVolume(). + Name("pv-with-read-write-once-pod-2"). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). + Capacity(storage.Requests). + HostPathVolumeSource(&v1.HostPathVolumeSource{Path: "/mnt2", Type: &volType}). + Obj() + pvc2 := st.MakePersistentVolumeClaim(). + Name("pvc-with-read-write-once-pod-2"). + Namespace(testCtx.NS.Name). + // Annotation and volume name required for PVC to be considered bound. + Annotation(volume.AnnBindCompleted, "true"). + VolumeName(pv2.Name). + AccessModes([]v1.PersistentVolumeAccessMode{v1.ReadWriteOncePod}). + Resources(storage). + Obj() + + tests := []struct { + name string + init func() error + existingPods []*v1.Pod + pod *v1.Pod + unresolvable bool + preemptedPodIndexes map[int]struct{} + cleanup func() error + }{ + { + name: "preempt single pod", + init: func() error { + _, err := testutils.CreatePV(cs, pv1) + if err != nil { + return fmt.Errorf("cannot create pv: %v", err) + } + _, err = testutils.CreatePVC(cs, pvc1) + if err != nil { + return fmt.Errorf("cannot create pvc: %v", err) + } + return nil + }, + existingPods: []*v1.Pod{ + initPausePod(&testutils.PausePodConfig{ + Name: "victim-pod", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Volumes: []v1.Volume{{ + Name: "volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc1.Name, + }, + }, + }}, + }), + }, + pod: initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Volumes: []v1.Volume{{ + Name: "volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc1.Name, + }, + }, + }}, + }), + preemptedPodIndexes: map[int]struct{}{0: {}}, + cleanup: func() error { + if err := testutils.DeletePVC(cs, pvc1.Name, pvc1.Namespace); err != nil { + return fmt.Errorf("cannot delete pvc: %v", err) + } + if err := testutils.DeletePV(cs, pv1.Name); err != nil { + return fmt.Errorf("cannot delete pv: %v", err) + } + return nil + }, + }, + { + name: "preempt two pods", + init: func() error { + for _, pv := range []*v1.PersistentVolume{pv1, pv2} { + _, err := testutils.CreatePV(cs, pv) + if err != nil { + return fmt.Errorf("cannot create pv: %v", err) + } + } + for _, pvc := range []*v1.PersistentVolumeClaim{pvc1, pvc2} { + _, err := testutils.CreatePVC(cs, pvc) + if err != nil { + return fmt.Errorf("cannot create pvc: %v", err) + } + } + return nil + }, + existingPods: []*v1.Pod{ + initPausePod(&testutils.PausePodConfig{ + Name: "victim-pod-1", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Volumes: []v1.Volume{{ + Name: "volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc1.Name, + }, + }, + }}, + }), + initPausePod(&testutils.PausePodConfig{ + Name: "victim-pod-2", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Volumes: []v1.Volume{{ + Name: "volume", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc2.Name, + }, + }, + }}, + }), + }, + pod: initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Volumes: []v1.Volume{ + { + Name: "volume-1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc1.Name, + }, + }, + }, + { + Name: "volume-2", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc2.Name, + }, + }, + }, + }, + }), + preemptedPodIndexes: map[int]struct{}{0: {}, 1: {}}, + cleanup: func() error { + for _, pvc := range []*v1.PersistentVolumeClaim{pvc1, pvc2} { + if err := testutils.DeletePVC(cs, pvc.Name, pvc.Namespace); err != nil { + return fmt.Errorf("cannot delete pvc: %v", err) + } + } + for _, pv := range []*v1.PersistentVolume{pv1, pv2} { + if err := testutils.DeletePV(cs, pv.Name); err != nil { + return fmt.Errorf("cannot delete pv: %v", err) + } + } + return nil + }, + }, + { + name: "preempt single pod with two volumes", + init: func() error { + for _, pv := range []*v1.PersistentVolume{pv1, pv2} { + _, err := testutils.CreatePV(cs, pv) + if err != nil { + return fmt.Errorf("cannot create pv: %v", err) + } + } + for _, pvc := range []*v1.PersistentVolumeClaim{pvc1, pvc2} { + _, err := testutils.CreatePVC(cs, pvc) + if err != nil { + return fmt.Errorf("cannot create pvc: %v", err) + } + } + return nil + }, + existingPods: []*v1.Pod{ + initPausePod(&testutils.PausePodConfig{ + Name: "victim-pod", + Namespace: testCtx.NS.Name, + Priority: &lowPriority, + Volumes: []v1.Volume{ + { + Name: "volume-1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc1.Name, + }, + }, + }, + { + Name: "volume-2", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc2.Name, + }, + }, + }, + }, + }), + }, + pod: initPausePod(&testutils.PausePodConfig{ + Name: "preemptor-pod", + Namespace: testCtx.NS.Name, + Priority: &highPriority, + Volumes: []v1.Volume{ + { + Name: "volume-1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc1.Name, + }, + }, + }, + { + Name: "volume-2", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc2.Name, + }, + }, + }, + }, + }), + preemptedPodIndexes: map[int]struct{}{0: {}}, + cleanup: func() error { + for _, pvc := range []*v1.PersistentVolumeClaim{pvc1, pvc2} { + if err := testutils.DeletePVC(cs, pvc.Name, pvc.Namespace); err != nil { + return fmt.Errorf("cannot delete pvc: %v", err) + } + } + for _, pv := range []*v1.PersistentVolume{pv1, pv2} { + if err := testutils.DeletePV(cs, pv.Name); err != nil { + return fmt.Errorf("cannot delete pv: %v", err) + } + } + return nil + }, + }, + } + + // Create a node with some resources and a label. + nodeRes := map[v1.ResourceName]string{ + v1.ResourcePods: "32", + v1.ResourceCPU: "500m", + v1.ResourceMemory: "500", + } + nodeObject := st.MakeNode().Name("node1").Capacity(nodeRes).Label("node", "node1").Obj() + if _, err := createNode(cs, nodeObject); err != nil { + t.Fatalf("Error creating node: %v", err) + } + + for _, asyncPreemptionEnabled := range []bool{true, false} { + for _, clearingNominatedNodeNameAfterBinding := range []bool{true, false} { + for _, test := range tests { + t.Run(fmt.Sprintf("%s (Async preemption enabled: %v, ClearingNominatedNodeNameAfterBinding: %v)", test.name, asyncPreemptionEnabled, clearingNominatedNodeNameAfterBinding), func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerAsyncPreemption, asyncPreemptionEnabled) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ClearingNominatedNodeNameAfterBinding, clearingNominatedNodeNameAfterBinding) + + if err := test.init(); err != nil { + t.Fatalf("Error while initializing test: %v", err) + } + + pods := make([]*v1.Pod, len(test.existingPods)) + t.Cleanup(func() { + testutils.CleanupPods(testCtx.Ctx, cs, t, pods) + if err := test.cleanup(); err != nil { + t.Errorf("Error cleaning up test: %v", err) + } + }) + // Create and run existingPods. + for i, p := range test.existingPods { + var err error + pods[i], err = runPausePod(cs, p) + if err != nil { + t.Fatalf("Error running pause pod: %v", err) + } + } + // Create the "pod". + preemptor, err := createPausePod(cs, test.pod) + if err != nil { + t.Errorf("Error while creating high priority pod: %v", err) + } + pods = append(pods, preemptor) + // Wait for preemption of pods and make sure the other ones are not preempted. + for i, p := range pods { + if _, found := test.preemptedPodIndexes[i]; found { + if err = wait.PollUntilContextTimeout(testCtx.Ctx, time.Second, wait.ForeverTestTimeout, false, + podIsGettingEvicted(cs, p.Namespace, p.Name)); err != nil { + t.Errorf("Pod %v/%v is not getting evicted.", p.Namespace, p.Name) + } + } else { + if p.DeletionTimestamp != nil { + t.Errorf("Didn't expect pod %v to get preempted.", p.Name) + } + } + } + // Also check that the preemptor pod gets the NominatedNodeName field set. + if len(test.preemptedPodIndexes) > 0 && !clearingNominatedNodeNameAfterBinding { + if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, preemptor); err != nil { + t.Errorf("NominatedNodeName field was not set for pod %v: %v", preemptor.Name, err) + } + } + }) + } + } + } +} diff --git a/test/integration/volume/persistent_volumes_test.go b/test/integration/volume/persistent_volumes_test.go index d3a9e6c346359..eed2c6526282c 100644 --- a/test/integration/volume/persistent_volumes_test.go +++ b/test/integration/volume/persistent_volumes_test.go @@ -1468,30 +1468,19 @@ func waitForAnyPersistentVolumePhase(w watch.Interface, phase v1.PersistentVolum } func waitForSomePersistentVolumeClaimPhase(ctx context.Context, testClient clientset.Interface, namespace string, objCount int, watchPVC watch.Interface, phase v1.PersistentVolumeClaimPhase) error { - // The caller doesn't know about new watches, so we have to stop them ourselves. - var newWatchPVC watch.Interface - defer func() { - if newWatchPVC != nil { - newWatchPVC.Stop() - } - }() - return wait.ExponentialBackoffWithContext(ctx, retry.DefaultBackoff, func(ctx context.Context) (bool, error) { for i := 0; i < objCount; i++ { - err := waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound) - if err != nil { - klog.Errorf("Failed to wait for a claim (%d/%d) to be bound: %v", i+1, objCount, err) + waitErr := waitForAnyPersistentVolumeClaimPhase(watchPVC, v1.ClaimBound) + if waitErr != nil { + klog.Errorf("Failed to wait for a claim (%d/%d) to be bound: %v", i+1, objCount, waitErr) klog.Info("Recreating a watch for claims and re-checking the count of bound claims") - newWatchPVC, err = testClient.CoreV1().PersistentVolumeClaims(namespace).Watch(ctx, metav1.ListOptions{}) + newWatchPVC, err := testClient.CoreV1().PersistentVolumeClaims(namespace).Watch(ctx, metav1.ListOptions{}) if err != nil { return false, err } watchPVC.Stop() watchPVC = newWatchPVC - // The error from waitForAnyPersistentVolumeClaimPhase is assumed to be harmless - // and/or to be fixed by recreating the watch, so here we continue polling - // via ExponentialBackoffWithContext. - return false, nil + return false, waitErr } klog.V(1).Infof("%d claims bound", i+1) } diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 242b286f947b8..2498c3c732515 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -221,7 +221,7 @@ func initImageConfigs(list RegistryList) (map[ImageID]Config, map[ImageID]Config configs[APIServer] = Config{list.PromoterE2eRegistry, "sample-apiserver", "1.29.2"} configs[AppArmorLoader] = Config{list.PromoterE2eRegistry, "apparmor-loader", "1.4"} configs[BusyBox] = Config{list.PromoterE2eRegistry, "busybox", "1.37.0-1"} - configs[DistrolessIptables] = Config{list.BuildImageRegistry, "distroless-iptables", "v0.7.11"} + configs[DistrolessIptables] = Config{list.BuildImageRegistry, "distroless-iptables", "v0.7.8"} configs[Etcd] = Config{list.GcEtcdRegistry, "etcd", "3.6.4-0"} configs[Httpd] = Config{list.PromoterE2eRegistry, "httpd", "2.4.38-4"} configs[HttpdNew] = Config{list.PromoterE2eRegistry, "httpd", "2.4.39-4"} diff --git a/vendor/k8s.io/system-validators/validators/types_unix.go b/vendor/k8s.io/system-validators/validators/types_unix.go index e2fc25fc5073d..28eb1d38ba89b 100644 --- a/vendor/k8s.io/system-validators/validators/types_unix.go +++ b/vendor/k8s.io/system-validators/validators/types_unix.go @@ -42,6 +42,12 @@ var DefaultSysSpec = SysSpec{ {Name: "IPC_NS"}, {Name: "UTS_NS"}, {Name: "CGROUPS"}, + {Name: "CGROUP_BPF"}, // cgroups v2 + {Name: "CGROUP_CPUACCT"}, // cgroups v1 cpuacct + {Name: "CGROUP_DEVICE"}, + {Name: "CGROUP_FREEZER"}, // cgroups v1 freezer + {Name: "CGROUP_PIDS"}, + {Name: "CGROUP_SCHED"}, // cgroups v1 & v2 cpu {Name: "CPUSETS"}, {Name: "MEMCG"}, {Name: "INET"}, @@ -56,6 +62,7 @@ var DefaultSysSpec = SysSpec{ {Name: "AUFS_FS", Description: "Required for aufs."}, {Name: "BLK_DEV_DM", Description: "Required for devicemapper."}, {Name: "CFS_BANDWIDTH", Description: "Required for CPU quota."}, + {Name: "CGROUP_HUGETLB", Description: "Required for hugetlb cgroup."}, {Name: "SECCOMP", Description: "Required for seccomp."}, {Name: "SECCOMP_FILTER", Description: "Required for seccomp mode 2."}, }, diff --git a/vendor/modules.txt b/vendor/modules.txt index 1130105273a0d..aff9ef6ea919d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1449,7 +1449,7 @@ k8s.io/kube-openapi/pkg/validation/validate ## explicit; go 1.24.0 # k8s.io/sample-apiserver v0.0.0 => ./staging/src/k8s.io/sample-apiserver ## explicit; go 1.24.0 -# k8s.io/system-validators v1.10.2 +# k8s.io/system-validators v1.10.1 ## explicit; go 1.16 k8s.io/system-validators/validators # k8s.io/utils v0.0.0-20250604170112-4c0f3b243397