diff --git a/CHANGELOG/CHANGELOG-1.21.md b/CHANGELOG/CHANGELOG-1.21.md index 5ce452b060c0a..a118ba5f7d211 100644 --- a/CHANGELOG/CHANGELOG-1.21.md +++ b/CHANGELOG/CHANGELOG-1.21.md @@ -1,18 +1,33 @@ -- [v1.21.0](#v1210) - - [Downloads for v1.21.0](#downloads-for-v1210) +- [v1.21.1](#v1211) + - [Downloads for v1.21.1](#downloads-for-v1211) + - [Source Code](#source-code) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) + - [Changelog since v1.21.0](#changelog-since-v1210) + - [Changes by Kind](#changes-by-kind) + - [API Change](#api-change) + - [Feature](#feature) + - [Failing Test](#failing-test) + - [Bug or Regression](#bug-or-regression) + - [Dependencies](#dependencies) + - [Added](#added) + - [Changed](#changed) + - [Removed](#removed) +- [v1.21.0](#v1210) + - [Downloads for v1.21.0](#downloads-for-v1210) + - [Source Code](#source-code-1) + - [Client Binaries](#client-binaries-1) + - [Server Binaries](#server-binaries-1) + - [Node Binaries](#node-binaries-1) - [Changelog since v1.20.0](#changelog-since-v1200) -- [Release notes for v1.21.0-rc.0](#release-notes-for-v1210-rc0) -- [Changelog since v1.20.0](#changelog-since-v1200-1) - [What's New (Major Themes)](#whats-new-major-themes) - [Deprecation of PodSecurityPolicy](#deprecation-of-podsecuritypolicy) - [Kubernetes API Reference Documentation](#kubernetes-api-reference-documentation) - [Kustomize Updates in Kubectl](#kustomize-updates-in-kubectl) - - [Default Container Labels](#default-container-labels) + - [Default Container Annotation](#default-container-annotation) - [Immutable Secrets and ConfigMaps](#immutable-secrets-and-configmaps) - [Structured Logging in Kubelet](#structured-logging-in-kubelet) - [Storage Capacity Tracking](#storage-capacity-tracking) @@ -23,36 +38,32 @@ - [TopologyAwareHints feature falls back to default behavior](#-feature-falls-back-to-default-behavior) - [Urgent Upgrade Notes](#urgent-upgrade-notes) - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade) - - [Changes by Kind](#changes-by-kind) + - [Changes by Kind](#changes-by-kind-1) - [Deprecation](#deprecation) - - [API Change](#api-change) - - [Feature](#feature) + - [API Change](#api-change-1) + - [Feature](#feature-1) - [Documentation](#documentation) - - [Failing Test](#failing-test) - - [Bug or Regression](#bug-or-regression) + - [Failing Test](#failing-test-1) + - [Bug or Regression](#bug-or-regression-1) - [Other (Cleanup or Flake)](#other-cleanup-or-flake) - [Uncategorized](#uncategorized) - - [Dependencies](#dependencies) - - [Added](#added) - - [Changed](#changed) - - [Removed](#removed) - [Dependencies](#dependencies-1) - [Added](#added-1) - [Changed](#changed-1) - [Removed](#removed-1) - [v1.21.0-rc.0](#v1210-rc0) - [Downloads for v1.21.0-rc.0](#downloads-for-v1210-rc0) - - [Source Code](#source-code) - - [Client binaries](#client-binaries-1) - - [Server binaries](#server-binaries-1) - - [Node binaries](#node-binaries-1) + - [Source Code](#source-code-2) + - [Client binaries](#client-binaries-2) + - [Server binaries](#server-binaries-2) + - [Node binaries](#node-binaries-2) - [Changelog since v1.21.0-beta.1](#changelog-since-v1210-beta1) - [Urgent Upgrade Notes](#urgent-upgrade-notes-1) - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-1) - - [Changes by Kind](#changes-by-kind-1) - - [API Change](#api-change-1) - - [Feature](#feature-1) - - [Bug or Regression](#bug-or-regression-1) + - [Changes by Kind](#changes-by-kind-2) + - [API Change](#api-change-2) + - [Feature](#feature-2) + - [Bug or Regression](#bug-or-regression-2) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-1) - [Dependencies](#dependencies-2) - [Added](#added-2) @@ -60,18 +71,18 @@ - [Removed](#removed-2) - [v1.21.0-beta.1](#v1210-beta1) - [Downloads for v1.21.0-beta.1](#downloads-for-v1210-beta1) - - [Source Code](#source-code-1) - - [Client binaries](#client-binaries-2) - - [Server binaries](#server-binaries-2) - - [Node binaries](#node-binaries-2) + - [Source Code](#source-code-3) + - [Client binaries](#client-binaries-3) + - [Server binaries](#server-binaries-3) + - [Node binaries](#node-binaries-3) - [Changelog since v1.21.0-beta.0](#changelog-since-v1210-beta0) - [Urgent Upgrade Notes](#urgent-upgrade-notes-2) - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-2) - - [Changes by Kind](#changes-by-kind-2) + - [Changes by Kind](#changes-by-kind-3) - [Deprecation](#deprecation-1) - - [API Change](#api-change-2) - - [Feature](#feature-2) - - [Bug or Regression](#bug-or-regression-2) + - [API Change](#api-change-3) + - [Feature](#feature-3) + - [Bug or Regression](#bug-or-regression-3) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-2) - [Uncategorized](#uncategorized-1) - [Dependencies](#dependencies-3) @@ -80,20 +91,20 @@ - [Removed](#removed-3) - [v1.21.0-beta.0](#v1210-beta0) - [Downloads for v1.21.0-beta.0](#downloads-for-v1210-beta0) - - [Source Code](#source-code-2) - - [Client binaries](#client-binaries-3) - - [Server binaries](#server-binaries-3) - - [Node binaries](#node-binaries-3) + - [Source Code](#source-code-4) + - [Client binaries](#client-binaries-4) + - [Server binaries](#server-binaries-4) + - [Node binaries](#node-binaries-4) - [Changelog since v1.21.0-alpha.3](#changelog-since-v1210-alpha3) - [Urgent Upgrade Notes](#urgent-upgrade-notes-3) - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-3) - - [Changes by Kind](#changes-by-kind-3) + - [Changes by Kind](#changes-by-kind-4) - [Deprecation](#deprecation-2) - - [API Change](#api-change-3) - - [Feature](#feature-3) + - [API Change](#api-change-4) + - [Feature](#feature-4) - [Documentation](#documentation-1) - - [Failing Test](#failing-test-1) - - [Bug or Regression](#bug-or-regression-3) + - [Failing Test](#failing-test-2) + - [Bug or Regression](#bug-or-regression-4) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-3) - [Dependencies](#dependencies-4) - [Added](#added-4) @@ -101,19 +112,19 @@ - [Removed](#removed-4) - [v1.21.0-alpha.3](#v1210-alpha3) - [Downloads for v1.21.0-alpha.3](#downloads-for-v1210-alpha3) - - [Source Code](#source-code-3) - - [Client binaries](#client-binaries-4) - - [Server binaries](#server-binaries-4) - - [Node binaries](#node-binaries-4) + - [Source Code](#source-code-5) + - [Client binaries](#client-binaries-5) + - [Server binaries](#server-binaries-5) + - [Node binaries](#node-binaries-5) - [Changelog since v1.21.0-alpha.2](#changelog-since-v1210-alpha2) - [Urgent Upgrade Notes](#urgent-upgrade-notes-4) - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-4) - - [Changes by Kind](#changes-by-kind-4) - - [API Change](#api-change-4) - - [Feature](#feature-4) + - [Changes by Kind](#changes-by-kind-5) + - [API Change](#api-change-5) + - [Feature](#feature-5) - [Documentation](#documentation-2) - - [Failing Test](#failing-test-2) - - [Bug or Regression](#bug-or-regression-4) + - [Failing Test](#failing-test-3) + - [Bug or Regression](#bug-or-regression-5) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-4) - [Dependencies](#dependencies-5) - [Added](#added-5) @@ -121,18 +132,18 @@ - [Removed](#removed-5) - [v1.21.0-alpha.2](#v1210-alpha2) - [Downloads for v1.21.0-alpha.2](#downloads-for-v1210-alpha2) - - [Source Code](#source-code-4) - - [Client binaries](#client-binaries-5) - - [Server binaries](#server-binaries-5) - - [Node binaries](#node-binaries-5) + - [Source Code](#source-code-6) + - [Client binaries](#client-binaries-6) + - [Server binaries](#server-binaries-6) + - [Node binaries](#node-binaries-6) - [Changelog since v1.21.0-alpha.1](#changelog-since-v1210-alpha1) - [Urgent Upgrade Notes](#urgent-upgrade-notes-5) - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-5) - - [Changes by Kind](#changes-by-kind-5) + - [Changes by Kind](#changes-by-kind-6) - [Deprecation](#deprecation-3) - - [API Change](#api-change-5) + - [API Change](#api-change-6) - [Documentation](#documentation-3) - - [Bug or Regression](#bug-or-regression-5) + - [Bug or Regression](#bug-or-regression-6) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-5) - [Dependencies](#dependencies-6) - [Added](#added-6) @@ -140,18 +151,18 @@ - [Removed](#removed-6) - [v1.21.0-alpha.1](#v1210-alpha1) - [Downloads for v1.21.0-alpha.1](#downloads-for-v1210-alpha1) - - [Source Code](#source-code-5) - - [Client binaries](#client-binaries-6) - - [Server binaries](#server-binaries-6) - - [Node binaries](#node-binaries-6) - - [Changelog since v1.20.0](#changelog-since-v1200-2) + - [Source Code](#source-code-7) + - [Client binaries](#client-binaries-7) + - [Server binaries](#server-binaries-7) + - [Node binaries](#node-binaries-7) + - [Changelog since v1.20.0](#changelog-since-v1200-1) - [Urgent Upgrade Notes](#urgent-upgrade-notes-6) - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-6) - - [Changes by Kind](#changes-by-kind-6) + - [Changes by Kind](#changes-by-kind-7) - [Deprecation](#deprecation-4) - - [API Change](#api-change-6) - - [Feature](#feature-5) - - [Bug or Regression](#bug-or-regression-6) + - [API Change](#api-change-7) + - [Feature](#feature-6) + - [Bug or Regression](#bug-or-regression-7) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-6) - [Uncategorized](#uncategorized-2) - [Dependencies](#dependencies-7) @@ -161,12 +172,121 @@ +# v1.21.1 + + +## Downloads for v1.21.1 + +### Source Code + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes.tar.gz) | 39005d4e11684d732babf419e06403a67ffb4f6c78cb7cb6e8214a7a1e6171de2a302d20d72bbb9da7f2748f46fd3bcf7d7e3b4767e3c8167f536d93e5f7bd7b +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-src.tar.gz) | 2d25debd4a4559ca9338c7f35682ec72d8aa5c1ebe424b9866fc9391fbbbc21dac5498b5af35c5109cbdfb5cee9a2259b99eb3086d05cff6510b9ac1407626df + +### Client Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-darwin-amd64.tar.gz) | b5ee08ce233f80f3c9d8918347169752df49f9745792762893b3bbc5c7a36246028533a405c8a41172205e3b6d54895ddc56e9b464feab7054f8ef5a2b2b059b +[kubernetes-client-darwin-arm64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-darwin-arm64.tar.gz) | 3e10c84c3cb184bf3d81bc62009e85dc9f6df00d381136617a71007c2b4758cc3879b4bb90b7e416d11d4e58d7a44eb89562640a2a75a1b369241a79068999be +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-linux-386.tar.gz) | 34391ca524b35d2f970ffd2f59becb4f8131b6576976af4724212afbdb972449deca87ee6bdcd4f10c8093fd0513a5d11baa59112a29338ec3e717ad93998a7f +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-linux-amd64.tar.gz) | 97050c54cd0f41e2efc954400bf65726c699b456506a0cbbb752aa79ec8662335dffa81c5373fb2e3231bab1e8ff73a3a1b5329caa2225ae7809fe1269c03f99 +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-linux-arm.tar.gz) | 2bb268e38004bde0ed27ac0c2a2fdbe5fe2c2a95272ef2f289c53049e24ee0b94bf4590fdfec99a1e8b72dfec917bd7a40435cbbda445fe04c9a6aad61f250a5 +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-linux-arm64.tar.gz) | ef67fc5795c95f3b9d8dab2476805167af5b3b84242fd5f340e73bb144a99040462093a6db53719dead419b77efd6e37660f513fcd93e870dc47f75e039dbbbb +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-linux-ppc64le.tar.gz) | 701e54ee5d3f0754c88ca766c3ce624157a2e96b57a72ea8a5e350e2ed4ac8302e9940b851bbfb93904629afa80f71438f6cf8d1110b9cb47efe99690830d967 +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-linux-s390x.tar.gz) | d44519897d65308d47a621a5995938379bdf022bf69ed5b3468d09b04800970e61e57d03acd0d48411fa9cade6c7df638c576710f414f00b716e54933ce4244c +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-windows-386.tar.gz) | a404d61d60e14182c931819512e5048f2ba3d51381a1b14fe52c54c49abc828aa32cadc4c377291d52151f5eca1b89d526389a12eed51ac353498f494b646522 +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-client-windows-amd64.tar.gz) | 0474acd68f2d298e772af003631d437fb4c7a445955f740389e01096910b20639a45d160122082020917fa187b754be396d64b3275cbab4aabdc451cc978a743 + +### Server Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-server-linux-amd64.tar.gz) | 9e370a48ef2a428ad3b0878d0ae0989caf156e36a2b40bf803aaddeed941c09120900eedc21e643d6e819529308e36db3da57bc887766f45ef2c8f56897a1170 +[kubernetes-server-linux-arm.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-server-linux-arm.tar.gz) | 1a6e256ba0a3318b1a419aac17eaabce9df9650a70974da0b677fcc6b628f87473394dc45b0524733424d2eaf2f09c0b9ac9baee5a9247a0d1376f026c202f38 +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-server-linux-arm64.tar.gz) | 24b146b1befbcc845b4e95e7b24d9d9cfc06bdf77409135ce6c7c0c99f4803ee9aed7f3cc99023ec0a9069f9f9044bd5b2562b13cb5825ff4317675ef160a7e7 +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-server-linux-ppc64le.tar.gz) | 36708c18c4760f2766245fd8c58038d1f21d118b0ccf50183a154e5a1c299a5f04db5c6b19d295b649c68e9ed320372253272314b306eda6bd674a715033548e +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-server-linux-s390x.tar.gz) | 3e9333253875306e055c00c0cced7d564c512807ada0ad499524b3a11b3fb4cfbca944c711b41ce9ebb2a631487208aa0543ed64e0d2e5e992d405a4f4c5990c + +### Node Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-node-linux-amd64.tar.gz) | 752bfccbc40b710a6042c4d590a8bf43b14c9dbf6d1afd1f5d87bd11465b45279d0d7508227eaedf148a9ebda4fe32106acf38d7becee1e844ba045ae7e8985e +[kubernetes-node-linux-arm.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-node-linux-arm.tar.gz) | 892781b14d2750c007133cc43cb12cdd7ccce9f03b28b40f4f840484b170349900b2016f7c3797f9df7f078dbd409606cf4f339cbd981ff64d0112c651cc1725 +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-node-linux-arm64.tar.gz) | af95314feb5085e0a13b271b78754c617f93846d67149931d6cd8d93d816e51cbf556b146b0992e6596998f7bbd31865801689a7cc1f6ee9654f743c5949a1a1 +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-node-linux-ppc64le.tar.gz) | 382512bfe522544d8f537133d4c62e2f976a1e6ccca7aaecbb8d974b839fba2bda4d4392de4096c9de83cc51586fe8c8454884e5e95020f562629fb3dbd618e3 +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-node-linux-s390x.tar.gz) | 14060d0ce6bb640ed6af389f1f20bc0219a59d5090eaac308f3aee6e0aed1236e786e015505bbed4c293de89560757b521a1b222b2ee87f57ad090fe26a67e0a +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.21.1/kubernetes-node-windows-amd64.tar.gz) | c72121f132107ebddf7a55a412e28f83633ffc78038c35e85dbe7eed253acf3370e56fe304b842c15a1476116f4a0a2cf1e92185b54d2176a79b6d914e93deab + +## Changelog since v1.21.0 + +## Changes by Kind + +### API Change + +- "Auto" is now a valid value for the `service.kubernetes.io/topology-aware-hints` annotation. ([#100728](https://github.com/kubernetes/kubernetes/pull/100728), [@robscott](https://github.com/robscott)) [SIG Apps, Instrumentation and Network] +- We have added a new Priority & Fairness rule that exempts all probes (/readyz, /healthz, /livez) to prevent + restarting of "healthy" kube-apiserver instance(s) by kubelet. ([#101111](https://github.com/kubernetes/kubernetes/pull/101111), [@tkashem](https://github.com/tkashem)) [SIG API Machinery] + +### Feature + +- Add NeedResize function to kubernetes/mount-utils, user can call this function to determine if fs need to be resized ([#101253](https://github.com/kubernetes/kubernetes/pull/101253), [@AndyXiangLi](https://github.com/AndyXiangLi)) [SIG Storage] +- Base image updates to mitigate kube-proxy and etcd container image CVEs + - debian-base to buster-v1.6.0 + - debian-iptables to buster-v1.6.0 ([#100976](https://github.com/kubernetes/kubernetes/pull/100976), [@jindijamie](https://github.com/jindijamie)) [SIG Release and Testing] +- Kubernetes is now built with Golang 1.16.3 ([#101209](https://github.com/kubernetes/kubernetes/pull/101209), [@justaugustus](https://github.com/justaugustus)) [SIG Cloud Provider, Instrumentation, Release and Testing] +- Kubernetes is now built with Golang 1.16.4 ([#101811](https://github.com/kubernetes/kubernetes/pull/101811), [@justaugustus](https://github.com/justaugustus)) [SIG Cloud Provider, Instrumentation, Release and Testing] + +### Failing Test + +- Fixed generic ephemeal volumes with OwnerReferencesPermissionEnforcement admission plugin enabled. ([#101186](https://github.com/kubernetes/kubernetes/pull/101186), [@jsafrane](https://github.com/jsafrane)) [SIG Auth and Storage] +- Resolves an issue with the "ServiceAccountIssuerDiscovery should support OIDC discovery" conformance test failing on clusters which are configured with issuers outside the cluster ([#101725](https://github.com/kubernetes/kubernetes/pull/101725), [@mtaufen](https://github.com/mtaufen)) [SIG Auth and Testing] + +### Bug or Regression + +- Adds node event handlers to dual stack kube-proxy implementation to fix Topology Aware Hints. ([#101054](https://github.com/kubernetes/kubernetes/pull/101054), [@robscott](https://github.com/robscott)) [SIG Network] +- Azurefile: Normalize share name to not include capital letters ([#100731](https://github.com/kubernetes/kubernetes/pull/100731), [@kassarl](https://github.com/kassarl)) [SIG Cloud Provider and Storage] +- EndpointSlice IP validation now matches Endpoints IP validation. ([#101084](https://github.com/kubernetes/kubernetes/pull/101084), [@robscott](https://github.com/robscott)) [SIG Apps and Network] +- Ensure service deleted when the Azure resource group has been deleted ([#100944](https://github.com/kubernetes/kubernetes/pull/100944), [@feiskyer](https://github.com/feiskyer)) [SIG Cloud Provider] +- Fix EndpointSlice describe panic when an Endpoint doesn't have zone ([#101025](https://github.com/kubernetes/kubernetes/pull/101025), [@tnqn](https://github.com/tnqn)) [SIG CLI] +- Fix display of Job completion mode on kubectl describe ([#101198](https://github.com/kubernetes/kubernetes/pull/101198), [@alculquicondor](https://github.com/alculquicondor)) [SIG CLI] +- Fix: azure file inline volume namespace issue in csi migration translation ([#101235](https://github.com/kubernetes/kubernetes/pull/101235), [@andyzhangx](https://github.com/andyzhangx)) [SIG Apps, Cloud Provider, Node and Storage] +- Fix: set "host is down" as corrupted mount ([#101398](https://github.com/kubernetes/kubernetes/pull/101398), [@andyzhangx](https://github.com/andyzhangx)) [SIG Cloud Provider and Storage] +- Fixed a bug where startupProbe stopped working after a container's first restart ([#101093](https://github.com/kubernetes/kubernetes/pull/101093), [@wzshiming](https://github.com/wzshiming)) [SIG Node] +- Fixed port-forward memory leak for long-running and heavily used connections. ([#99839](https://github.com/kubernetes/kubernetes/pull/99839), [@saschagrunert](https://github.com/saschagrunert)) [SIG API Machinery and Node] +- Kubectl create service now respects namespace flag ([#101005](https://github.com/kubernetes/kubernetes/pull/101005), [@zxh326](https://github.com/zxh326)) [SIG CLI] +- Kubelet: improve the performance when waiting for a synchronization of the node list with the kube-apiserver ([#99336](https://github.com/kubernetes/kubernetes/pull/99336), [@neolit123](https://github.com/neolit123)) [SIG Node] +- No support endpointslice in linux userpace mode ([#101504](https://github.com/kubernetes/kubernetes/pull/101504), [@JornShen](https://github.com/JornShen)) [SIG Network] +- Renames the timeout field for the DelegatingAuthenticationOptions to TokenRequestTimeout and set the timeout only for the token review client. Previously the timeout was also applied to watches making them reconnecting every 10 seconds. ([#101102](https://github.com/kubernetes/kubernetes/pull/101102), [@p0lyn0mial](https://github.com/p0lyn0mial)) [SIG API Machinery, Auth and Cloud Provider] +- Respect ExecProbeTimeout=false for dockershim ([#101127](https://github.com/kubernetes/kubernetes/pull/101127), [@jackfrancis](https://github.com/jackfrancis)) [SIG Node and Testing] +- Upgrades functionality of `kubectl kustomize` as described at https://github.com/kubernetes-sigs/kustomize/releases/tag/kustomize%2Fv4.1.0 ([#101177](https://github.com/kubernetes/kubernetes/pull/101177), [@KnVerey](https://github.com/KnVerey)) [SIG CLI] + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +- sigs.k8s.io/kustomize/api: v0.8.5 → v0.8.8 +- sigs.k8s.io/kustomize/cmd/config: v0.9.7 → v0.9.10 +- sigs.k8s.io/kustomize/kustomize/v4: v4.0.5 → v4.1.2 +- sigs.k8s.io/kustomize/kyaml: v0.10.15 → v0.10.17 + +### Removed +_Nothing has changed._ + + + # v1.21.0 [Documentation](https://docs.k8s.io) ## Downloads for v1.21.0 +### Source Code + filename | sha512 hash -------- | ----------- [kubernetes.tar.gz](https://dl.k8s.io/v1.21.0/kubernetes.tar.gz) | `19bb76a3fa5ce4b9f043b2a3a77c32365ab1fcb902d8dd6678427fb8be8f49f64a5a03dc46aaef9c7dadee05501cf83412eda46f0edacbb8fc1ed0bf5fb79142` @@ -210,12 +330,6 @@ filename | sha512 hash ## Changelog since v1.20.0 -# Release notes for v1.21.0-rc.0 - -[Documentation](https://docs.k8s.io/docs/home) - -# Changelog since v1.20.0 - ## What's New (Major Themes) ### Deprecation of PodSecurityPolicy @@ -230,9 +344,9 @@ The API reference is now generated with [`gen-resourcesdocs`](https://github.com [Kustomize](https://github.com/kubernetes-sigs/kustomize) version in kubectl had a jump from v2.0.3 to [v4.0.5](https://github.com/kubernetes/kubernetes/pull/98946). Kustomize is now treated as a library and future updates will be less sporadic. -### Default Container Labels +### Default Container Annotation -Pod with multiple containers can use `kubectl.kubernetes.io/default-container` label to have a container preselected for kubectl commands. More can be read in [KEP-2227](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cli/2227-kubectl-default-container/README.md). +Pod with multiple containers can use `kubectl.kubernetes.io/default-container` annotation to have a container preselected for kubectl commands. More can be read in [KEP-2227](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cli/2227-kubectl-default-container/README.md). ### Immutable Secrets and ConfigMaps @@ -702,91 +816,6 @@ The feature gate currently falls back to the default behavior in most cases. Ena - sigs.k8s.io/kustomize: v2.0.3+incompatible -## Dependencies - -### Added -- github.com/go-errors/errors: [v1.0.1](https://github.com/go-errors/errors/tree/v1.0.1) -- github.com/gobuffalo/here: [v0.6.0](https://github.com/gobuffalo/here/tree/v0.6.0) -- github.com/google/shlex: [e7afc7f](https://github.com/google/shlex/tree/e7afc7f) -- github.com/markbates/pkger: [v0.17.1](https://github.com/markbates/pkger/tree/v0.17.1) -- github.com/moby/spdystream: [v0.2.0](https://github.com/moby/spdystream/tree/v0.2.0) -- github.com/monochromegane/go-gitignore: [205db1a](https://github.com/monochromegane/go-gitignore/tree/205db1a) -- github.com/niemeyer/pretty: [a10e7ca](https://github.com/niemeyer/pretty/tree/a10e7ca) -- github.com/xlab/treeprint: [a009c39](https://github.com/xlab/treeprint/tree/a009c39) -- go.starlark.net: 8dd3e2e -- golang.org/x/term: 6a3ed07 -- sigs.k8s.io/kustomize/api: v0.8.5 -- sigs.k8s.io/kustomize/cmd/config: v0.9.7 -- sigs.k8s.io/kustomize/kustomize/v4: v4.0.5 -- sigs.k8s.io/kustomize/kyaml: v0.10.15 - -### Changed -- dmitri.shuralyov.com/gpu/mtl: 666a987 → 28db891 -- github.com/Azure/go-autorest/autorest: [v0.11.1 → v0.11.12](https://github.com/Azure/go-autorest/autorest/compare/v0.11.1...v0.11.12) -- github.com/NYTimes/gziphandler: [56545f4 → v1.1.1](https://github.com/NYTimes/gziphandler/compare/56545f4...v1.1.1) -- github.com/cilium/ebpf: [1c8d4c9 → v0.2.0](https://github.com/cilium/ebpf/compare/1c8d4c9...v0.2.0) -- github.com/container-storage-interface/spec: [v1.2.0 → v1.3.0](https://github.com/container-storage-interface/spec/compare/v1.2.0...v1.3.0) -- github.com/containerd/console: [v1.0.0 → v1.0.1](https://github.com/containerd/console/compare/v1.0.0...v1.0.1) -- github.com/containerd/containerd: [v1.4.1 → v1.4.4](https://github.com/containerd/containerd/compare/v1.4.1...v1.4.4) -- github.com/coredns/corefile-migration: [v1.0.10 → v1.0.11](https://github.com/coredns/corefile-migration/compare/v1.0.10...v1.0.11) -- github.com/creack/pty: [v1.1.7 → v1.1.11](https://github.com/creack/pty/compare/v1.1.7...v1.1.11) -- github.com/docker/docker: [bd33bbf → v20.10.2+incompatible](https://github.com/docker/docker/compare/bd33bbf...v20.10.2) -- github.com/go-logr/logr: [v0.2.0 → v0.4.0](https://github.com/go-logr/logr/compare/v0.2.0...v0.4.0) -- github.com/go-openapi/spec: [v0.19.3 → v0.19.5](https://github.com/go-openapi/spec/compare/v0.19.3...v0.19.5) -- github.com/go-openapi/strfmt: [v0.19.3 → v0.19.5](https://github.com/go-openapi/strfmt/compare/v0.19.3...v0.19.5) -- github.com/go-openapi/validate: [v0.19.5 → v0.19.8](https://github.com/go-openapi/validate/compare/v0.19.5...v0.19.8) -- github.com/gogo/protobuf: [v1.3.1 → v1.3.2](https://github.com/gogo/protobuf/compare/v1.3.1...v1.3.2) -- github.com/golang/mock: [v1.4.1 → v1.4.4](https://github.com/golang/mock/compare/v1.4.1...v1.4.4) -- github.com/google/cadvisor: [v0.38.5 → v0.39.0](https://github.com/google/cadvisor/compare/v0.38.5...v0.39.0) -- github.com/heketi/heketi: [c2e2a4a → v10.2.0+incompatible](https://github.com/heketi/heketi/compare/c2e2a4a...v10.2.0) -- github.com/kisielk/errcheck: [v1.2.0 → v1.5.0](https://github.com/kisielk/errcheck/compare/v1.2.0...v1.5.0) -- github.com/konsorten/go-windows-terminal-sequences: [v1.0.3 → v1.0.2](https://github.com/konsorten/go-windows-terminal-sequences/compare/v1.0.3...v1.0.2) -- github.com/kr/text: [v0.1.0 → v0.2.0](https://github.com/kr/text/compare/v0.1.0...v0.2.0) -- github.com/mattn/go-runewidth: [v0.0.2 → v0.0.7](https://github.com/mattn/go-runewidth/compare/v0.0.2...v0.0.7) -- github.com/miekg/dns: [v1.1.4 → v1.1.35](https://github.com/miekg/dns/compare/v1.1.4...v1.1.35) -- github.com/moby/sys/mountinfo: [v0.1.3 → v0.4.0](https://github.com/moby/sys/mountinfo/compare/v0.1.3...v0.4.0) -- github.com/moby/term: [672ec06 → df9cb8a](https://github.com/moby/term/compare/672ec06...df9cb8a) -- github.com/mrunalp/fileutils: [abd8a0e → v0.5.0](https://github.com/mrunalp/fileutils/compare/abd8a0e...v0.5.0) -- github.com/olekukonko/tablewriter: [a0225b3 → v0.0.4](https://github.com/olekukonko/tablewriter/compare/a0225b3...v0.0.4) -- github.com/opencontainers/runc: [v1.0.0-rc92 → v1.0.0-rc93](https://github.com/opencontainers/runc/compare/v1.0.0-rc92...v1.0.0-rc93) -- github.com/opencontainers/runtime-spec: [4d89ac9 → e6143ca](https://github.com/opencontainers/runtime-spec/compare/4d89ac9...e6143ca) -- github.com/opencontainers/selinux: [v1.6.0 → v1.8.0](https://github.com/opencontainers/selinux/compare/v1.6.0...v1.8.0) -- github.com/sergi/go-diff: [v1.0.0 → v1.1.0](https://github.com/sergi/go-diff/compare/v1.0.0...v1.1.0) -- github.com/sirupsen/logrus: [v1.6.0 → v1.7.0](https://github.com/sirupsen/logrus/compare/v1.6.0...v1.7.0) -- github.com/syndtr/gocapability: [d983527 → 42c35b4](https://github.com/syndtr/gocapability/compare/d983527...42c35b4) -- github.com/willf/bitset: [d5bec33 → v1.1.11](https://github.com/willf/bitset/compare/d5bec33...v1.1.11) -- github.com/yuin/goldmark: [v1.1.27 → v1.2.1](https://github.com/yuin/goldmark/compare/v1.1.27...v1.2.1) -- golang.org/x/crypto: 7f63de1 → 5ea612d -- golang.org/x/exp: 6cc2880 → 85be41e -- golang.org/x/mobile: d2bd2a2 → e6ae53a -- golang.org/x/mod: v0.3.0 → ce943fd -- golang.org/x/net: 69a7880 → 3d97a24 -- golang.org/x/sync: cd5d95a → 67f06af -- golang.org/x/sys: 5cba982 → a50acf3 -- golang.org/x/time: 3af7569 → f8bda1e -- golang.org/x/tools: c1934b7 → v0.1.0 -- gopkg.in/check.v1: 41f04d3 → 8fa4692 -- gopkg.in/yaml.v2: v2.2.8 → v2.4.0 -- gotest.tools/v3: v3.0.2 → v3.0.3 -- k8s.io/gengo: 83324d8 → b6c5ce2 -- k8s.io/klog/v2: v2.4.0 → v2.8.0 -- k8s.io/kube-openapi: d219536 → 591a79e -- k8s.io/system-validators: v1.2.0 → v1.4.0 -- sigs.k8s.io/apiserver-network-proxy/konnectivity-client: v0.0.14 → v0.0.15 -- sigs.k8s.io/structured-merge-diff/v4: v4.0.2 → v4.1.0 - -### Removed -- github.com/codegangsta/negroni: [v1.0.0](https://github.com/codegangsta/negroni/tree/v1.0.0) -- github.com/docker/spdystream: [449fdfc](https://github.com/docker/spdystream/tree/449fdfc) -- github.com/golangplus/bytes: [45c989f](https://github.com/golangplus/bytes/tree/45c989f) -- github.com/golangplus/fmt: [2a5d6d7](https://github.com/golangplus/fmt/tree/2a5d6d7) -- github.com/gorilla/context: [v1.1.1](https://github.com/gorilla/context/tree/v1.1.1) -- github.com/kr/pty: [v1.1.5](https://github.com/kr/pty/tree/v1.1.5) -- rsc.io/quote/v3: v3.1.0 -- rsc.io/sampler: v1.3.0 -- sigs.k8s.io/kustomize: v2.0.3+incompatible - - # v1.21.0-rc.0 @@ -1119,7 +1148,7 @@ filename | sha512 hash - On single-stack configured (IPv4 or IPv6, but not both) clusters, Services which are both headless (no clusterIP) and selectorless (empty or undefined selector) will report `ipFamilyPolicy RequireDualStack` and will have entries in `ipFamilies[]` for both IPv4 and IPv6. This is a change from alpha, but does not have any impact on the manually-specified Endpoints and EndpointSlices for the Service. ([#99555](https://github.com/kubernetes/kubernetes/pull/99555), [@thockin](https://github.com/thockin)) [SIG Apps and Network] - Resolves spurious `Failed to list *v1.Secret` or `Failed to list *v1.ConfigMap` messages in kubelet logs. ([#99538](https://github.com/kubernetes/kubernetes/pull/99538), [@liggitt](https://github.com/liggitt)) [SIG Auth and Node] - Return zero time (midnight on Jan. 1, 1970) instead of negative number when reporting startedAt and finishedAt of the not started or a running Pod when using dockershim as a runtime. ([#99585](https://github.com/kubernetes/kubernetes/pull/99585), [@Iceber](https://github.com/Iceber)) [SIG Node] -- Stdin is now only passed to client-go exec credential plugins when it is detected to be an interactive terminal. Previously, it was passed to client-go exec plugins when **stdout*- was detected to be an interactive terminal. ([#99654](https://github.com/kubernetes/kubernetes/pull/99654), [@ankeesler](https://github.com/ankeesler)) [SIG API Machinery and Auth] +- Stdin is now only passed to client-go exec credential plugins when it is detected to be an interactive terminal. Previously, it was passed to client-go exec plugins when **stdout** was detected to be an interactive terminal. ([#99654](https://github.com/kubernetes/kubernetes/pull/99654), [@ankeesler](https://github.com/ankeesler)) [SIG API Machinery and Auth] - The maximum number of ports allowed in EndpointSlices has been increased from 100 to 20,000 ([#99795](https://github.com/kubernetes/kubernetes/pull/99795), [@robscott](https://github.com/robscott)) [SIG Network] - Updates the commands - kubectl kustomize {arg} @@ -1282,7 +1311,7 @@ filename | sha512 hash - A new histogram metric to track the time it took to delete a job by the ttl-after-finished controller ([#98676](https://github.com/kubernetes/kubernetes/pull/98676), [@ahg-g](https://github.com/ahg-g)) [SIG Apps and Instrumentation] - AWS cloudprovider supports auto-discovering subnets without any kubernetes.io/cluster/ tags. It also supports additional service annotation service.beta.kubernetes.io/aws-load-balancer-subnets to manually configure the subnets. ([#97431](https://github.com/kubernetes/kubernetes/pull/97431), [@kishorj](https://github.com/kishorj)) [SIG Cloud Provider] -- Add --permit-address-sharing flag to kube-apiserver to listen with SO_REUSEADDR. While allowing to listen on wildcard IPs like 0.0.0.0 and specific IPs in parallel, it avoid waiting for the kernel to release socket in TIME_WAIT state, and hence, considably reducing kube-apiserver restart times under certain conditions. ([#93861](https://github.com/kubernetes/kubernetes/pull/93861), [@sttts](https://github.com/sttts)) [SIG API Machinery] +- Add `--permit-address-sharing` flag to kube-apiserver to listen with SO_REUSEADDR. While allowing to listen on wildcard IPs like 0.0.0.0 and specific IPs in parallel, it avoid waiting for the kernel to release socket in TIME_WAIT state, and hence, considably reducing kube-apiserver restart times under certain conditions. ([#93861](https://github.com/kubernetes/kubernetes/pull/93861), [@sttts](https://github.com/sttts)) [SIG API Machinery] - Add `csi_operations_seconds` metric on kubelet that exposes CSI operations duration and status for node CSI operations. ([#98979](https://github.com/kubernetes/kubernetes/pull/98979), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Instrumentation and Storage] - Add `migrated` field into `storage_operation_duration_seconds` metric ([#99050](https://github.com/kubernetes/kubernetes/pull/99050), [@Jiawei0227](https://github.com/Jiawei0227)) [SIG Apps, Instrumentation and Storage] - Add bash-completion for comma separated list on `kubectl get` ([#98301](https://github.com/kubernetes/kubernetes/pull/98301), [@phil9909](https://github.com/phil9909)) [SIG CLI] @@ -1308,7 +1337,7 @@ filename | sha512 hash ### Bug or Regression -- Add missing --kube-api-content-type in kubemark hollow template ([#98911](https://github.com/kubernetes/kubernetes/pull/98911), [@Jeffwan](https://github.com/Jeffwan)) [SIG Scalability and Testing] +- Add missing `--kube-api-content-type` in kubemark hollow template ([#98911](https://github.com/kubernetes/kubernetes/pull/98911), [@Jeffwan](https://github.com/Jeffwan)) [SIG Scalability and Testing] - Avoid duplicate error messages when runing kubectl edit quota ([#98201](https://github.com/kubernetes/kubernetes/pull/98201), [@pacoxu](https://github.com/pacoxu)) [SIG API Machinery and Apps] - Cleanup subnet in frontend IP configs to prevent huge subnet request bodies in some scenarios. ([#98133](https://github.com/kubernetes/kubernetes/pull/98133), [@nilo19](https://github.com/nilo19)) [SIG Cloud Provider] - Fix errors when accessing Windows container stats for Dockershim ([#98510](https://github.com/kubernetes/kubernetes/pull/98510), [@jsturtevant](https://github.com/jsturtevant)) [SIG Node and Windows] @@ -1418,7 +1447,7 @@ filename | sha512 hash - Export NewDebuggingRoundTripper function and DebugLevel options in the k8s.io/client-go/transport package. ([#98324](https://github.com/kubernetes/kubernetes/pull/98324), [@atosatto](https://github.com/atosatto)) [SIG API Machinery] - Kubectl wait ensures that observedGeneration >= generation if applicable ([#97408](https://github.com/kubernetes/kubernetes/pull/97408), [@KnicKnic](https://github.com/KnicKnic)) [SIG CLI] - Kubernetes is now built using go1.15.8 ([#98834](https://github.com/kubernetes/kubernetes/pull/98834), [@cpanato](https://github.com/cpanato)) [SIG Cloud Provider, Instrumentation, Release and Testing] -- New admission controller "denyserviceexternalips" is available. Clusters which do not *need- the Service "externalIPs" feature should enable this controller and be more secure. ([#97395](https://github.com/kubernetes/kubernetes/pull/97395), [@thockin](https://github.com/thockin)) [SIG API Machinery] +- New admission controller "denyserviceexternalips" is available. Clusters which do not *need* the Service "externalIPs" feature should enable this controller and be more secure. ([#97395](https://github.com/kubernetes/kubernetes/pull/97395), [@thockin](https://github.com/thockin)) [SIG API Machinery] - Overall, enable the feature of `PreferNominatedNode` will improve the performance of scheduling where preemption might frequently happen, but in theory, enable the feature of `PreferNominatedNode`, the pod might not be scheduled to the best candidate node in the cluster. ([#93179](https://github.com/kubernetes/kubernetes/pull/93179), [@chendave](https://github.com/chendave)) [SIG Scheduling and Testing] - Pause image upgraded to 3.4.1 in kubelet and kubeadm for both Linux and Windows. ([#98205](https://github.com/kubernetes/kubernetes/pull/98205), [@pacoxu](https://github.com/pacoxu)) [SIG CLI, Cloud Provider, Cluster Lifecycle, Node, Testing and Windows] - The `ServiceAccountIssuerDiscovery` feature has graduated to GA, and is unconditionally enabled. The `ServiceAccountIssuerDiscovery` feature-gate will be removed in 1.22. ([#98553](https://github.com/kubernetes/kubernetes/pull/98553), [@mtaufen](https://github.com/mtaufen)) [SIG API Machinery, Auth and Testing] diff --git a/LICENSES/vendor/github.com/checkpoint-restore/go-criu/v4/LICENSE b/LICENSES/vendor/github.com/checkpoint-restore/go-criu/v5/LICENSE similarity index 98% rename from LICENSES/vendor/github.com/checkpoint-restore/go-criu/v4/LICENSE rename to LICENSES/vendor/github.com/checkpoint-restore/go-criu/v5/LICENSE index f0b0b31878561..029457c8ae949 100644 --- a/LICENSES/vendor/github.com/checkpoint-restore/go-criu/v4/LICENSE +++ b/LICENSES/vendor/github.com/checkpoint-restore/go-criu/v5/LICENSE @@ -1,4 +1,4 @@ -= vendor/github.com/checkpoint-restore/go-criu/v4 licensed under: = += vendor/github.com/checkpoint-restore/go-criu/v5 licensed under: = Apache License Version 2.0, January 2004 @@ -202,4 +202,4 @@ See the License for the specific language governing permissions and limitations under the License. -= vendor/github.com/checkpoint-restore/go-criu/v4/LICENSE e3fc50a88d0a364313df4b21ef20c29e += vendor/github.com/checkpoint-restore/go-criu/v5/LICENSE e3fc50a88d0a364313df4b21ef20c29e diff --git a/build/build-image/cross/VERSION b/build/build-image/cross/VERSION index ef32e84b21e12..b0ae55d054e21 100644 --- a/build/build-image/cross/VERSION +++ b/build/build-image/cross/VERSION @@ -1 +1 @@ -v1.16.4-1 +v1.16.5-1 diff --git a/build/common.sh b/build/common.sh index f3cb40448c812..ac49130830366 100755 --- a/build/common.sh +++ b/build/common.sh @@ -86,8 +86,9 @@ readonly KUBE_RSYNC_PORT="${KUBE_RSYNC_PORT:-}" readonly KUBE_CONTAINER_RSYNC_PORT=8730 # These are the default versions (image tags) for their respective base images. -readonly __default_debian_iptables_version=buster-v1.6.0 -readonly __default_go_runner_version=v2.3.1-go1.16.4-buster.0 +readonly __default_debian_iptables_version=buster-v1.6.2 +readonly __default_go_runner_version=v2.3.1-go1.16.5-buster.0 +readonly __default_setcap_version=buster-v2.0.1 # These are the base images for the Docker-wrapped binaries. readonly KUBE_GORUNNER_IMAGE="${KUBE_GORUNNER_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/go-runner:$__default_go_runner_version}" @@ -97,7 +98,7 @@ readonly KUBE_SCHEDULER_BASE_IMAGE="${KUBE_SCHEDULER_BASE_IMAGE:-$KUBE_GORUNNER_ readonly KUBE_PROXY_BASE_IMAGE="${KUBE_PROXY_BASE_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/debian-iptables:$__default_debian_iptables_version}" # This is the image used in a multi-stage build to apply capabilities to Docker-wrapped binaries. -readonly KUBE_BUILD_SETCAP_IMAGE="${KUBE_BUILD_SETCAP_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/setcap:buster-v1.4.0}" +readonly KUBE_BUILD_SETCAP_IMAGE="${KUBE_BUILD_SETCAP_IMAGE:-$KUBE_BASE_IMAGE_REGISTRY/setcap:$__default_setcap_version}" # Get the set of master binaries that run in Docker (on Linux) # Entry format is ",". diff --git a/build/dependencies.yaml b/build/dependencies.yaml index 93b9148e2f52c..45cbedd9ceb94 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -101,7 +101,7 @@ dependencies: # Golang - name: "golang: upstream version" - version: 1.16.4 + version: 1.16.5 refPaths: - path: build/build-image/cross/VERSION - path: cluster/addons/fluentd-elasticsearch/es-image/Dockerfile @@ -124,7 +124,7 @@ dependencies: match: minimum_go_version=go([0-9]+\.[0-9]+) - name: "k8s.gcr.io/kube-cross: dependents" - version: v1.16.4-1 + version: v1.16.5-1 refPaths: - path: build/build-image/cross/VERSION - path: test/images/sample-apiserver/Makefile @@ -132,7 +132,7 @@ dependencies: # Base images - name: "k8s.gcr.io/debian-base: dependents" - version: buster-v1.6.0 + version: buster-v1.7.0 refPaths: - path: cluster/images/etcd/Makefile match: BASEIMAGE\?\=k8s\.gcr\.io\/build-image\/debian-base:[a-zA-Z]+\-v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?) @@ -146,7 +146,7 @@ dependencies: match: BASEIMAGE\?\=k8s\.gcr\.io\/build-image\/debian-base-s390x:[a-zA-Z]+\-v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?) - name: "k8s.gcr.io/debian-iptables: dependents" - version: buster-v1.6.0 + version: buster-v1.6.2 refPaths: - path: build/common.sh match: __default_debian_iptables_version= @@ -154,7 +154,7 @@ dependencies: match: configs\[DebianIptables\] = Config{buildImageRegistry, "debian-iptables", "[a-zA-Z]+\-v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)"} - name: "k8s.gcr.io/go-runner: dependents" - version: v2.3.1-go1.16.4-buster.0 + version: v2.3.1-go1.16.5-buster.0 refPaths: - path: build/common.sh match: __default_go_runner_version= @@ -217,6 +217,12 @@ dependencies: - path: test/utils/image/manifest.go match: configs\[Pause\] = Config{gcRegistry, "pause", "\d+\.\d+.\d+"} + - name: "k8s.gcr.io/setcap: dependents" + version: buster-v2.0.1 + refPaths: + - path: build/common.sh + match: __default_setcap_version= + # metadata-concealment: bump this one first - name: "metadata-concealment" version: "1.6" diff --git a/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile b/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile index 674dd3f5c6e21..5e18184203a91 100644 --- a/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile +++ b/cluster/addons/fluentd-elasticsearch/es-image/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.16.4 AS builder +FROM golang:1.16.5 AS builder COPY elasticsearch_logging_discovery.go go.mod go.sum / RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -a -ldflags "-w" -o /elasticsearch_logging_discovery /elasticsearch_logging_discovery.go diff --git a/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml b/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml index ea859edefcd66..22e5e2716e7fb 100644 --- a/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml +++ b/cluster/gce/addons/konnectivity-agent/konnectivity-agent-ds.yaml @@ -25,7 +25,7 @@ spec: nodeSelector: kubernetes.io/os: linux containers: - - image: k8s.gcr.io/kas-network-proxy/proxy-agent:v0.0.15 + - image: k8s.gcr.io/kas-network-proxy/proxy-agent:v0.0.19 name: konnectivity-agent command: ["/proxy-agent"] args: [ diff --git a/cluster/gce/manifests/konnectivity-server.yaml b/cluster/gce/manifests/konnectivity-server.yaml index b50b507b15baf..62626b46754ca 100644 --- a/cluster/gce/manifests/konnectivity-server.yaml +++ b/cluster/gce/manifests/konnectivity-server.yaml @@ -13,7 +13,7 @@ spec: hostNetwork: true containers: - name: konnectivity-server-container - image: k8s.gcr.io/kas-network-proxy/proxy-server:v0.0.15 + image: k8s.gcr.io/kas-network-proxy/proxy-server:v0.0.19 resources: requests: cpu: 25m diff --git a/cluster/get-kube-binaries.sh b/cluster/get-kube-binaries.sh index dc9d29a6537ed..589a335f01572 100755 --- a/cluster/get-kube-binaries.sh +++ b/cluster/get-kube-binaries.sh @@ -158,7 +158,7 @@ function download_tarball() { mkdir -p "${download_path}" if [[ $(which gsutil) ]] && [[ "$url" =~ ^https://storage.googleapis.com/.* ]]; then - gsutil cp "${url//'https://storage.googleapis.com/'/'gs://'}" "${download_path}/${file}" + gsutil cp "${url//'https://storage.googleapis.com/'/gs://}" "${download_path}/${file}" elif [[ $(which curl) ]]; then # if the url belongs to GCS API we should use oauth2_token in the headers curl_headers="" diff --git a/cluster/get-kube.sh b/cluster/get-kube.sh index f6a3419885e30..d3075f687e6a9 100755 --- a/cluster/get-kube.sh +++ b/cluster/get-kube.sh @@ -241,7 +241,7 @@ fi if "${need_download}"; then if [[ $(which gsutil) ]] && [[ "$kubernetes_tar_url" =~ ^https://storage.googleapis.com/.* ]]; then - gsutil cp "${kubernetes_tar_url//'https://storage.googleapis.com/'/'gs://'}" "${file}" + gsutil cp "${kubernetes_tar_url//'https://storage.googleapis.com/'/gs://}" "${file}" elif [[ $(which curl) ]]; then # if the url belongs to GCS API we should use oauth2_token in the headers curl_headers="" diff --git a/cluster/images/etcd/Makefile b/cluster/images/etcd/Makefile index f67f031a0dce1..156980c70e4da 100644 --- a/cluster/images/etcd/Makefile +++ b/cluster/images/etcd/Makefile @@ -31,7 +31,7 @@ BUNDLED_ETCD_VERSIONS?=3.0.17 3.1.12 3.2.24 3.3.17 3.4.13 # LATEST_ETCD_VERSION identifies the most recent etcd version available. LATEST_ETCD_VERSION?=3.4.13 -# REVISION provides a version number fo this image and all it's bundled +# REVISION provides a version number for this image and all it's bundled # artifacts. It should start at zero for each LATEST_ETCD_VERSION and increment # for each revision of this image at that etcd version. REVISION?=4 @@ -67,19 +67,19 @@ GOARM?=7 TEMP_DIR:=$(shell mktemp -d) ifeq ($(ARCH),amd64) - BASEIMAGE?=k8s.gcr.io/build-image/debian-base:buster-v1.6.0 + BASEIMAGE?=k8s.gcr.io/build-image/debian-base:buster-v1.7.0 endif ifeq ($(ARCH),arm) - BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm:buster-v1.6.0 + BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm:buster-v1.7.0 endif ifeq ($(ARCH),arm64) - BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm64:buster-v1.6.0 + BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm64:buster-v1.7.0 endif ifeq ($(ARCH),ppc64le) - BASEIMAGE?=k8s.gcr.io/build-image/debian-base-ppc64le:buster-v1.6.0 + BASEIMAGE?=k8s.gcr.io/build-image/debian-base-ppc64le:buster-v1.7.0 endif ifeq ($(ARCH),s390x) - BASEIMAGE?=k8s.gcr.io/build-image/debian-base-s390x:buster-v1.6.0 + BASEIMAGE?=k8s.gcr.io/build-image/debian-base-s390x:buster-v1.7.0 endif RUNNERIMAGE?=gcr.io/distroless/static:latest diff --git a/cmd/kubeadm/app/constants/constants.go b/cmd/kubeadm/app/constants/constants.go index 3cb2d11ad4559..aed3a71302076 100644 --- a/cmd/kubeadm/app/constants/constants.go +++ b/cmd/kubeadm/app/constants/constants.go @@ -328,7 +328,7 @@ const ( CoreDNSDeploymentName = "coredns" // CoreDNSImageName specifies the name of the image for CoreDNS add-on - CoreDNSImageName = "coredns/coredns" + CoreDNSImageName = "coredns" // CoreDNSVersion is the version of CoreDNS to be deployed if it is used CoreDNSVersion = "v1.8.0" diff --git a/cmd/kubeadm/app/images/images.go b/cmd/kubeadm/app/images/images.go index edf087e9c464e..7ada3b750183a 100644 --- a/cmd/kubeadm/app/images/images.go +++ b/cmd/kubeadm/app/images/images.go @@ -21,6 +21,7 @@ import ( "k8s.io/klog/v2" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" "k8s.io/kubernetes/cmd/kubeadm/app/constants" kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util" ) @@ -52,6 +53,10 @@ func GetDNSImage(cfg *kubeadmapi.ClusterConfiguration) string { if cfg.DNS.ImageRepository != "" { dnsImageRepository = cfg.DNS.ImageRepository } + // Handle the renaming of the official image from "k8s.gcr.io/coredns" to "k8s.gcr.io/coredns/coredns + if dnsImageRepository == kubeadmapiv1beta2.DefaultImageRepository { + dnsImageRepository = fmt.Sprintf("%s/coredns", dnsImageRepository) + } // DNS uses an imageTag that corresponds to the DNS version matching the Kubernetes version dnsImageTag := constants.GetDNSVersion(cfg.DNS.Type) diff --git a/cmd/kubeadm/app/images/images_test.go b/cmd/kubeadm/app/images/images_test.go index f5165406e0fe6..e5b417bcdc77a 100644 --- a/cmd/kubeadm/app/images/images_test.go +++ b/cmd/kubeadm/app/images/images_test.go @@ -22,6 +22,7 @@ import ( "testing" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2" "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) @@ -237,3 +238,52 @@ func TestGetAllImages(t *testing.T) { }) } } + +func TestGetDNSImage(t *testing.T) { + var tests = []struct { + expected string + cfg *kubeadmapi.ClusterConfiguration + }{ + { + expected: "foo.io/coredns:v1.8.0", + cfg: &kubeadmapi.ClusterConfiguration{ + ImageRepository: "foo.io", + DNS: kubeadmapi.DNS{ + Type: kubeadmapi.CoreDNS, + }, + }, + }, + { + expected: kubeadmapiv1beta2.DefaultImageRepository + "/coredns/coredns:v1.8.0", + cfg: &kubeadmapi.ClusterConfiguration{ + ImageRepository: kubeadmapiv1beta2.DefaultImageRepository, + DNS: kubeadmapi.DNS{ + Type: kubeadmapi.CoreDNS, + }, + }, + }, + { + expected: "foo.io/coredns/coredns:v1.8.0", + cfg: &kubeadmapi.ClusterConfiguration{ + ImageRepository: "foo.io", + DNS: kubeadmapi.DNS{ + Type: kubeadmapi.CoreDNS, + ImageMeta: kubeadmapi.ImageMeta{ + ImageRepository: "foo.io/coredns", + }, + }, + }, + }, + } + + for _, test := range tests { + actual := GetDNSImage(test.cfg) + if actual != test.expected { + t.Errorf( + "failed to GetDNSImage:\n\texpected: %s\n\t actual: %s", + test.expected, + actual, + ) + } + } +} diff --git a/cmd/kubeadm/app/phases/etcd/local.go b/cmd/kubeadm/app/phases/etcd/local.go index 5cdcb35394135..c7fbc704419ab 100644 --- a/cmd/kubeadm/app/phases/etcd/local.go +++ b/cmd/kubeadm/app/phases/etcd/local.go @@ -236,9 +236,8 @@ func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A }, Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("100Mi"), + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), }, }, LivenessProbe: staticpodutil.LivenessProbe(probeHostname, "/health", probePort, probeScheme), diff --git a/cmd/kubelet/app/server_linux.go b/cmd/kubelet/app/server_linux.go index c22e24d5312fc..3eef6f13d9e2b 100644 --- a/cmd/kubelet/app/server_linux.go +++ b/cmd/kubelet/app/server_linux.go @@ -29,6 +29,7 @@ func watchForLockfileContention(path string, done chan struct{}) error { } if err = watcher.AddWatch(path, inotify.InOpen|inotify.InDeleteSelf); err != nil { klog.ErrorS(err, "Unable to watch lockfile") + watcher.Close() return err } go func() { @@ -39,6 +40,7 @@ func watchForLockfileContention(path string, done chan struct{}) error { klog.ErrorS(err, "inotify watcher error") } close(done) + watcher.Close() }() return nil } diff --git a/go.mod b/go.mod index 3ecbf4db1c06e..565ab1c25d56a 100644 --- a/go.mod +++ b/go.mod @@ -69,10 +69,10 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/runc v1.0.0-rc95.0.20210608002938-1f5126fe967e github.com/opencontainers/selinux v1.8.0 - github.com/openshift/api v0.0.0-20210521075222-e273a339932a + github.com/openshift/api v0.0.0-20210610130314-a6ac319a7eed github.com/openshift/apiserver-library-go v0.0.0-20210521113822-91c23a9a7ddf github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 - github.com/openshift/library-go v0.0.0-20210521084623-7392ea9b02ca + github.com/openshift/library-go v0.0.0-20210611143017-0d0ef669a361 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_model v0.2.0 @@ -133,6 +133,7 @@ require ( k8s.io/sample-apiserver v0.0.0 k8s.io/system-validators v1.4.0 k8s.io/utils v0.0.0-20210521133846-da695404a2bc + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.1.0 sigs.k8s.io/yaml v1.2.0 ) @@ -189,7 +190,6 @@ replace ( github.com/census-instrumentation/opencensus-proto => github.com/census-instrumentation/opencensus-proto v0.2.1 github.com/cespare/xxhash/v2 => github.com/cespare/xxhash/v2 v2.1.1 github.com/chai2010/gettext-go => github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 - github.com/checkpoint-restore/go-criu/v4 => github.com/checkpoint-restore/go-criu/v4 v4.1.0 github.com/checkpoint-restore/go-criu/v5 => github.com/checkpoint-restore/go-criu/v5 v5.0.0 github.com/cheekybits/genny => github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9 github.com/chzyer/logex => github.com/chzyer/logex v1.1.10 @@ -547,7 +547,7 @@ replace ( modernc.org/strutil => modernc.org/strutil v1.0.0 modernc.org/xc => modernc.org/xc v1.0.0 rsc.io/pdf => rsc.io/pdf v0.1.1 - sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 + sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 sigs.k8s.io/controller-tools => sigs.k8s.io/controller-tools v0.2.8 sigs.k8s.io/kube-storage-version-migrator => sigs.k8s.io/kube-storage-version-migrator v0.0.3 sigs.k8s.io/kustomize/api => sigs.k8s.io/kustomize/api v0.8.8 diff --git a/go.sum b/go.sum index 6820f8cd83e1d..bbc320e9dfa54 100644 --- a/go.sum +++ b/go.sum @@ -642,8 +642,8 @@ modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 h1:0jaDAAxtqIrrqas4vtTqxct4xS5kHfRNycTRLTyJmVM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.8 h1:G2z6JPSSjtWWgMeWSoHdXqyftJNmMmyxXpwENGoOtGE= diff --git a/openshift-hack/images/hyperkube/Dockerfile.rhel b/openshift-hack/images/hyperkube/Dockerfile.rhel index 7d2166df594b5..a72e052fa5bc8 100644 --- a/openshift-hack/images/hyperkube/Dockerfile.rhel +++ b/openshift-hack/images/hyperkube/Dockerfile.rhel @@ -13,4 +13,4 @@ COPY --from=builder /tmp/build/* /usr/bin/ LABEL io.k8s.display-name="OpenShift Kubernetes Server Commands" \ io.k8s.description="OpenShift is a platform for developing, building, and deploying containerized applications." \ io.openshift.tags="openshift,hyperkube" \ - io.openshift.build.versions="kubernetes=1.21.1" + io.openshift.build.versions="kubernetes=1.21.2" diff --git a/pkg/controller/endpointslicemirroring/reconciler.go b/pkg/controller/endpointslicemirroring/reconciler.go index 168e2107230aa..ae5cf53eb1cf3 100644 --- a/pkg/controller/endpointslicemirroring/reconciler.go +++ b/pkg/controller/endpointslicemirroring/reconciler.go @@ -95,7 +95,7 @@ func (r *reconciler) reconcile(endpoints *corev1.Endpoints, existingSlices []*di if totalAddressesAdded >= int(r.maxEndpointsPerSubset) { break } - if ok := d.addAddress(address, multiKey, true); ok { + if ok := d.addAddress(address, multiKey, false); ok { totalAddressesAdded++ } else { numInvalidAddresses++ diff --git a/pkg/controller/endpointslicemirroring/reconciler_test.go b/pkg/controller/endpointslicemirroring/reconciler_test.go index 9b8b2d856be6f..7f5ff72886fb3 100644 --- a/pkg/controller/endpointslicemirroring/reconciler_test.go +++ b/pkg/controller/endpointslicemirroring/reconciler_test.go @@ -228,6 +228,67 @@ func TestReconcile(t *testing.T) { expectedNumSlices: 1, expectedClientActions: 1, expectedMetrics: &expectedMetrics{desiredSlices: 1, actualSlices: 1, desiredEndpoints: 2, addedPerSync: 2, numCreated: 1}, + }, { + testName: "Endpoints with 1 subset, 2 ports, and 2 not ready addresses", + subsets: []corev1.EndpointSubset{{ + Ports: []corev1.EndpointPort{{ + Name: "http", + Port: 80, + Protocol: corev1.ProtocolTCP, + }, { + Name: "https", + Port: 443, + Protocol: corev1.ProtocolUDP, + }}, + NotReadyAddresses: []corev1.EndpointAddress{{ + IP: "10.0.0.1", + Hostname: "pod-1", + NodeName: utilpointer.StringPtr("node-1"), + }, { + IP: "10.0.0.2", + Hostname: "pod-2", + NodeName: utilpointer.StringPtr("node-2"), + }}, + }}, + existingEndpointSlices: []*discovery.EndpointSlice{}, + expectedNumSlices: 1, + expectedClientActions: 1, + expectedMetrics: &expectedMetrics{desiredSlices: 1, actualSlices: 1, desiredEndpoints: 2, addedPerSync: 2, numCreated: 1}, + }, { + testName: "Endpoints with 1 subset, 2 ports, and 2 ready and 2 not ready addresses", + subsets: []corev1.EndpointSubset{{ + Ports: []corev1.EndpointPort{{ + Name: "http", + Port: 80, + Protocol: corev1.ProtocolTCP, + }, { + Name: "https", + Port: 443, + Protocol: corev1.ProtocolUDP, + }}, + Addresses: []corev1.EndpointAddress{{ + IP: "10.1.1.1", + Hostname: "pod-11", + NodeName: utilpointer.StringPtr("node-1"), + }, { + IP: "10.1.1.2", + Hostname: "pod-12", + NodeName: utilpointer.StringPtr("node-2"), + }}, + NotReadyAddresses: []corev1.EndpointAddress{{ + IP: "10.0.0.1", + Hostname: "pod-1", + NodeName: utilpointer.StringPtr("node-1"), + }, { + IP: "10.0.0.2", + Hostname: "pod-2", + NodeName: utilpointer.StringPtr("node-2"), + }}, + }}, + existingEndpointSlices: []*discovery.EndpointSlice{}, + expectedNumSlices: 1, + expectedClientActions: 1, + expectedMetrics: &expectedMetrics{desiredSlices: 1, actualSlices: 1, desiredEndpoints: 4, addedPerSync: 4, numCreated: 1}, }, { testName: "Endpoints with 2 subsets, multiple ports and addresses", subsets: []corev1.EndpointSubset{{ diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller.go b/pkg/controller/volume/attachdetach/attach_detach_controller.go index e7a05690c0177..ea37235d2e0fd 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller.go @@ -57,6 +57,7 @@ import ( "k8s.io/kubernetes/pkg/features" proxyutil "k8s.io/kubernetes/pkg/proxy/util" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csi" "k8s.io/kubernetes/pkg/volume/csimigration" volumeutil "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor" @@ -714,17 +715,45 @@ func (adc *attachDetachController) processVolumeAttachments() error { klog.Errorf("Unable to lookup pv object for: %q, err: %v", *pvName, err) continue } + + var plugin volume.AttachableVolumePlugin volumeSpec := volume.NewSpecFromPersistentVolume(pv, false) - plugin, err := adc.volumePluginMgr.FindAttachablePluginBySpec(volumeSpec) - if err != nil || plugin == nil { - // Currently VA objects are created for CSI volumes only. nil plugin is unexpected, generate a warning - klog.Warningf( - "Skipping processing the volume %q on nodeName: %q, no attacher interface found. err=%v", - *pvName, - nodeName, - err) - continue + + // Consult csiMigratedPluginManager first before querying the plugins registered during runtime in volumePluginMgr. + // In-tree plugins that provisioned PVs will not be registered anymore after migration to CSI, once the respective + // feature gate is enabled. + if inTreePluginName, err := adc.csiMigratedPluginManager.GetInTreePluginNameFromSpec(pv, nil); err == nil { + if adc.csiMigratedPluginManager.IsMigrationEnabledForPlugin(inTreePluginName) { + // PV is migrated and should be handled by the CSI plugin instead of the in-tree one + plugin, _ = adc.volumePluginMgr.FindAttachablePluginByName(csi.CSIPluginName) + // podNamespace is not needed here for Azurefile as the volumeName generated will be the same with or without podNamespace + volumeSpec, err = csimigration.TranslateInTreeSpecToCSI(volumeSpec, "" /* podNamespace */, adc.intreeToCSITranslator) + if err != nil { + klog.Errorf( + "Failed to translate intree volumeSpec to CSI volumeSpec for volume:%q, va.Name:%q, nodeName:%q: %s. Error: %v", + *pvName, + va.Name, + nodeName, + inTreePluginName, + err) + continue + } + } } + + if plugin == nil { + plugin, err = adc.volumePluginMgr.FindAttachablePluginBySpec(volumeSpec) + if err != nil || plugin == nil { + // Currently VA objects are created for CSI volumes only. nil plugin is unexpected, generate a warning + klog.Warningf( + "Skipping processing the volume %q on nodeName: %q, no attacher interface found. err=%v", + *pvName, + nodeName, + err) + continue + } + } + volumeName, err := volumeutil.GetUniqueVolumeNameFromSpec(plugin, volumeSpec) if err != nil { klog.Errorf( diff --git a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go index 36845402e8f3d..bcadae559dcf6 100644 --- a/pkg/controller/volume/attachdetach/attach_detach_controller_test.go +++ b/pkg/controller/volume/attachdetach/attach_detach_controller_test.go @@ -26,12 +26,22 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" kcache "k8s.io/client-go/tools/cache" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/controller" "k8s.io/kubernetes/pkg/controller/volume/attachdetach/cache" controllervolumetesting "k8s.io/kubernetes/pkg/controller/volume/attachdetach/testing" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/volume" + "k8s.io/kubernetes/pkg/volume/csi" + "k8s.io/kubernetes/pkg/volume/util" +) + +const ( + intreePDUniqueNamePrefix = "kubernetes.io/gce-pd/" + csiPDUniqueNamePrefix = "kubernetes.io/csi/pd.csi.storage.gke.io^projects/UNSPECIFIED/zones/UNSPECIFIED/disks/" ) func Test_NewAttachDetachController_Positive(t *testing.T) { @@ -340,16 +350,18 @@ func attachDetachRecoveryTestCase(t *testing.T, extraPods1 []*v1.Pod, extraPods2 } type vaTest struct { - testName string - volName string - podName string - podNodeName string - pvName string - vaName string - vaNodeName string - vaAttachStatus bool - expected_attaches map[string][]string - expected_detaches map[string][]string + testName string + volName string + podName string + podNodeName string + pvName string + vaName string + vaNodeName string + vaAttachStatus bool + csiMigration bool + expected_attaches map[string][]string + expected_detaches map[string][]string + expectedASWAttachState cache.AttachState } func Test_ADC_VolumeAttachmentRecovery(t *testing.T) { @@ -386,6 +398,27 @@ func Test_ADC_VolumeAttachmentRecovery(t *testing.T) { expected_attaches: map[string][]string{}, expected_detaches: map[string][]string{"mynode-1": {"vol1"}}, }, + { // pod is scheduled, volume is migrated, attach status:false, verify volume is marked as attached + testName: "Scheduled Pod with migrated PV", + volName: "vol1", + podNodeName: "mynode-1", + pvName: "pv1", + vaName: "va1", + vaNodeName: "mynode-1", + vaAttachStatus: false, + csiMigration: true, + expectedASWAttachState: cache.AttachStateAttached, + }, + { // pod is deleted, volume is migrated, attach status:false, verify volume is marked as uncertain + testName: "Deleted Pod with migrated PV", + volName: "vol1", + pvName: "pv1", + vaName: "va1", + vaNodeName: "mynode-1", + vaAttachStatus: false, + csiMigration: true, + expectedASWAttachState: cache.AttachStateUncertain, + }, } { t.Run(tc.testName, func(t *testing.T) { volumeAttachmentRecoveryTestCase(t, tc) @@ -396,7 +429,17 @@ func Test_ADC_VolumeAttachmentRecovery(t *testing.T) { func volumeAttachmentRecoveryTestCase(t *testing.T, tc vaTest) { fakeKubeClient := controllervolumetesting.CreateTestClient() informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, time.Second*1) - plugins := controllervolumetesting.CreateTestPlugin() + var plugins []volume.VolumePlugin + if tc.csiMigration { + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigration, tc.csiMigration)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIMigrationGCE, tc.csiMigration)() + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InTreePluginGCEUnregister, tc.csiMigration)() + + // if InTreePluginGCEUnregister is enabled, only the CSI plugin is registered but not the in-tree one + plugins = append(plugins, csi.ProbeVolumePlugins()...) + } else { + plugins = controllervolumetesting.CreateTestPlugin() + } nodeInformer := informerFactory.Core().V1().Nodes().Informer() podInformer := informerFactory.Core().V1().Pods().Informer() pvInformer := informerFactory.Core().V1().PersistentVolumes().Informer() @@ -443,6 +486,32 @@ func volumeAttachmentRecoveryTestCase(t *testing.T, tc vaTest) { nodeInformer.GetIndexer().Add(&nodeToAdd) } + if tc.csiMigration { + newNode := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: tc.podNodeName, + Labels: map[string]string{ + "name": tc.podNodeName, + }, + Annotations: map[string]string{ + util.ControllerManagedAttachAnnotation: "true", + }, + }, + Status: v1.NodeStatus{ + VolumesAttached: []v1.AttachedVolume{ + { + Name: v1.UniqueVolumeName(csiPDUniqueNamePrefix + tc.volName), + DevicePath: "fake/path", + }, + }, + }, + } + _, err = adc.kubeClient.CoreV1().Nodes().Update(context.TODO(), newNode, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("Run failed with error. Failed to create a new pod: <%v>", err) + } + nodeInformer.GetIndexer().Add(&newNode) + } // Create and add objects requested by the test if tc.podName != "" { newPod := controllervolumetesting.NewPodWithVolume(tc.podName, tc.volName, tc.podNodeName) @@ -461,7 +530,7 @@ func volumeAttachmentRecoveryTestCase(t *testing.T, tc vaTest) { pvInformer.GetIndexer().Add(newPv) } if tc.vaName != "" { - newVa := controllervolumetesting.NewVolumeAttachment("va1", "pv1", "mynode-1", false) + newVa := controllervolumetesting.NewVolumeAttachment(tc.vaName, tc.pvName, tc.vaNodeName, tc.vaAttachStatus) _, err = adc.kubeClient.StorageV1().VolumeAttachments().Create(context.TODO(), newVa, metav1.CreateOptions{}) if err != nil { t.Fatalf("Run failed with error. Failed to create a new volumeAttachment: <%v>", err) @@ -497,8 +566,34 @@ func volumeAttachmentRecoveryTestCase(t *testing.T, tc vaTest) { go adc.desiredStateOfWorldPopulator.Run(stopCh) defer close(stopCh) - // Verify if expected attaches and detaches have happened - testPlugin := plugins[0].(*controllervolumetesting.TestPlugin) + if tc.csiMigration { + verifyExpectedVolumeState(t, adc, tc) + } else { + // Verify if expected attaches and detaches have happened + testPlugin := plugins[0].(*controllervolumetesting.TestPlugin) + verifyAttachDetachCalls(t, testPlugin, tc) + } + +} + +func verifyExpectedVolumeState(t *testing.T, adc *attachDetachController, tc vaTest) { + // Since csi migration is turned on, the attach state for the PV should be in CSI format. + attachedState := adc.actualStateOfWorld.GetAttachState( + v1.UniqueVolumeName(csiPDUniqueNamePrefix+tc.volName), types.NodeName(tc.vaNodeName)) + if attachedState != tc.expectedASWAttachState { + t.Fatalf("Expected attachedState %v, but it is %v", tc.expectedASWAttachState, attachedState) + } + + // kubernetes.io/gce-pd/ should not be marked when CSI Migration is on + // so it should be in detach status + attachedState = adc.actualStateOfWorld.GetAttachState( + v1.UniqueVolumeName(intreePDUniqueNamePrefix+tc.volName), types.NodeName(tc.vaNodeName)) + if attachedState != cache.AttachStateDetached { + t.Fatalf("Expected attachedState not to be %v, but it is %v", cache.AttachStateDetached, attachedState) + } +} + +func verifyAttachDetachCalls(t *testing.T, testPlugin *controllervolumetesting.TestPlugin, tc vaTest) { for tries := 0; tries <= 10; tries++ { // wait & try few times before failing the test expected_op_map := tc.expected_attaches plugin_map := testPlugin.GetAttachedVolumes() @@ -543,5 +638,4 @@ func volumeAttachmentRecoveryTestCase(t *testing.T, tc vaTest) { if testPlugin.GetErrorEncountered() { t.Fatalf("Fatal error encountered in the testing volume plugin") } - } diff --git a/pkg/controller/volume/attachdetach/testing/testvolumespec.go b/pkg/controller/volume/attachdetach/testing/testvolumespec.go index 1a176f74fe7a1..3fd8ce9705773 100644 --- a/pkg/controller/volume/attachdetach/testing/testvolumespec.go +++ b/pkg/controller/volume/attachdetach/testing/testvolumespec.go @@ -158,6 +158,16 @@ func CreateTestClient() *fake.Clientset { } attachVolumeToNode("lostVolumeName", nodeName) } + fakeClient.AddReactor("update", "nodes", func(action core.Action) (handled bool, ret runtime.Object, err error) { + updateAction := action.(core.UpdateAction) + node := updateAction.GetObject().(*v1.Node) + for index, n := range nodes.Items { + if n.Name == node.Name { + nodes.Items[index] = *node + } + } + return true, updateAction.GetObject(), nil + }) fakeClient.AddReactor("list", "nodes", func(action core.Action) (handled bool, ret runtime.Object, err error) { obj := &v1.NodeList{} obj.Items = append(obj.Items, nodes.Items...) diff --git a/pkg/kubelet/cm/internal_container_lifecycle.go b/pkg/kubelet/cm/internal_container_lifecycle.go index 278f13eb08e2d..92b36c2f9af88 100644 --- a/pkg/kubelet/cm/internal_container_lifecycle.go +++ b/pkg/kubelet/cm/internal_container_lifecycle.go @@ -50,10 +50,7 @@ func (i *internalContainerLifecycleImpl) PreStartContainer(pod *v1.Pod, containe } if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.TopologyManager) { - err := i.topologyManager.AddContainer(pod, containerID) - if err != nil { - return err - } + i.topologyManager.AddContainer(pod, container, containerID) } return nil } diff --git a/pkg/kubelet/cm/memorymanager/policy_static_test.go b/pkg/kubelet/cm/memorymanager/policy_static_test.go index fd27c581323de..ef30457215cbe 100644 --- a/pkg/kubelet/cm/memorymanager/policy_static_test.go +++ b/pkg/kubelet/cm/memorymanager/policy_static_test.go @@ -83,9 +83,8 @@ func (m *fakeTopologyManagerWithHint) AddHintProvider(h topologymanager.HintProv m.t.Logf("[fake topologymanager] AddHintProvider HintProvider: %v", h) } -func (m *fakeTopologyManagerWithHint) AddContainer(pod *v1.Pod, containerID string) error { - m.t.Logf("[fake topologymanager] AddContainer pod: %v container id: %v", format.Pod(pod), containerID) - return nil +func (m *fakeTopologyManagerWithHint) AddContainer(pod *v1.Pod, container *v1.Container, containerID string) { + m.t.Logf("[fake topologymanager] AddContainer pod: %v container name: %v container id: %v", format.Pod(pod), container.Name, containerID) } func (m *fakeTopologyManagerWithHint) RemoveContainer(containerID string) error { diff --git a/pkg/kubelet/cm/topologymanager/fake_topology_manager.go b/pkg/kubelet/cm/topologymanager/fake_topology_manager.go index 063cac65a27d5..1e7a831108dd6 100644 --- a/pkg/kubelet/cm/topologymanager/fake_topology_manager.go +++ b/pkg/kubelet/cm/topologymanager/fake_topology_manager.go @@ -39,9 +39,8 @@ func (m *fakeManager) AddHintProvider(h HintProvider) { klog.InfoS("AddHintProvider", "hintProvider", h) } -func (m *fakeManager) AddContainer(pod *v1.Pod, containerID string) error { - klog.InfoS("AddContainer", "pod", klog.KObj(pod), "containerID", containerID) - return nil +func (m *fakeManager) AddContainer(pod *v1.Pod, container *v1.Container, containerID string) { + klog.InfoS("AddContainer", "pod", klog.KObj(pod), "containerName", container.Name, "containerID", containerID) } func (m *fakeManager) RemoveContainer(containerID string) error { diff --git a/pkg/kubelet/cm/topologymanager/fake_topology_manager_test.go b/pkg/kubelet/cm/topologymanager/fake_topology_manager_test.go index 8297474311fb4..517b265071232 100644 --- a/pkg/kubelet/cm/topologymanager/fake_topology_manager_test.go +++ b/pkg/kubelet/cm/topologymanager/fake_topology_manager_test.go @@ -21,7 +21,6 @@ import ( "testing" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/kubernetes/pkg/kubelet/lifecycle" ) @@ -57,36 +56,6 @@ func TestFakeGetAffinity(t *testing.T) { } } -func TestFakeAddContainer(t *testing.T) { - testCases := []struct { - name string - containerID string - podUID types.UID - }{ - { - name: "Case1", - containerID: "nginx", - podUID: "0aafa4c4-38e8-11e9-bcb1-a4bf01040474", - }, - { - name: "Case2", - containerID: "Busy_Box", - podUID: "b3ee37fc-39a5-11e9-bcb1-a4bf01040474", - }, - } - fm := fakeManager{} - for _, tc := range testCases { - pod := v1.Pod{} - pod.UID = tc.podUID - err := fm.AddContainer(&pod, tc.containerID) - if err != nil { - t.Errorf("Expected error to be nil but got: %v", err) - - } - - } -} - func TestFakeRemoveContainer(t *testing.T) { testCases := []struct { name string diff --git a/pkg/kubelet/cm/topologymanager/scope.go b/pkg/kubelet/cm/topologymanager/scope.go index af90663368a47..c5c6f36be9727 100644 --- a/pkg/kubelet/cm/topologymanager/scope.go +++ b/pkg/kubelet/cm/topologymanager/scope.go @@ -23,6 +23,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/lifecycle" ) @@ -43,7 +44,7 @@ type Scope interface { // wants to be consoluted with when making topology hints AddHintProvider(h HintProvider) // AddContainer adds pod to Manager for tracking - AddContainer(pod *v1.Pod, containerID string) error + AddContainer(pod *v1.Pod, container *v1.Container, containerID string) // RemoveContainer removes pod from Manager tracking RemoveContainer(containerID string) error // Store is the interface for storing pod topology hints @@ -60,8 +61,8 @@ type scope struct { hintProviders []HintProvider // Topology Manager Policy policy Policy - // Mapping of PodUID to ContainerID for Adding/Removing Pods from PodTopologyHints mapping - podMap map[string]string + // Mapping of (PodUid, ContainerName) to ContainerID for Adding/Removing Pods from PodTopologyHints mapping + podMap containermap.ContainerMap } func (s *scope) Name() string { @@ -94,12 +95,11 @@ func (s *scope) AddHintProvider(h HintProvider) { // It would be better to implement this function in topologymanager instead of scope // but topologymanager do not track mapping anymore -func (s *scope) AddContainer(pod *v1.Pod, containerID string) error { +func (s *scope) AddContainer(pod *v1.Pod, container *v1.Container, containerID string) { s.mutex.Lock() defer s.mutex.Unlock() - s.podMap[containerID] = string(pod.UID) - return nil + s.podMap.Add(string(pod.UID), container.Name, containerID) } // It would be better to implement this function in topologymanager instead of scope @@ -109,10 +109,18 @@ func (s *scope) RemoveContainer(containerID string) error { defer s.mutex.Unlock() klog.InfoS("RemoveContainer", "containerID", containerID) - podUIDString := s.podMap[containerID] - delete(s.podMap, containerID) - if _, exists := s.podTopologyHints[podUIDString]; exists { - delete(s.podTopologyHints[podUIDString], containerID) + // Get the podUID and containerName associated with the containerID to be removed and remove it + podUIDString, containerName, err := s.podMap.GetContainerRef(containerID) + if err != nil { + return nil + } + s.podMap.RemoveByContainerID(containerID) + + // In cases where a container has been restarted, it's possible that the same podUID and + // containerName are already associated with a *different* containerID now. Only remove + // the TopologyHints associated with that podUID and containerName if this is not true + if _, err := s.podMap.GetContainerID(podUIDString, containerName); err != nil { + delete(s.podTopologyHints[podUIDString], containerName) if len(s.podTopologyHints[podUIDString]) == 0 { delete(s.podTopologyHints, podUIDString) } diff --git a/pkg/kubelet/cm/topologymanager/scope_container.go b/pkg/kubelet/cm/topologymanager/scope_container.go index e5d331e00e927..de45209625a64 100644 --- a/pkg/kubelet/cm/topologymanager/scope_container.go +++ b/pkg/kubelet/cm/topologymanager/scope_container.go @@ -19,6 +19,7 @@ package topologymanager import ( "k8s.io/api/core/v1" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/lifecycle" ) @@ -36,7 +37,7 @@ func NewContainerScope(policy Policy) Scope { name: containerTopologyScope, podTopologyHints: podTopologyHints{}, policy: policy, - podMap: make(map[string]string), + podMap: containermap.NewContainerMap(), }, } } diff --git a/pkg/kubelet/cm/topologymanager/scope_pod.go b/pkg/kubelet/cm/topologymanager/scope_pod.go index f4645bc4d768d..9ccc6414dd9f6 100644 --- a/pkg/kubelet/cm/topologymanager/scope_pod.go +++ b/pkg/kubelet/cm/topologymanager/scope_pod.go @@ -19,6 +19,7 @@ package topologymanager import ( "k8s.io/api/core/v1" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/lifecycle" ) @@ -36,7 +37,7 @@ func NewPodScope(policy Policy) Scope { name: podTopologyScope, podTopologyHints: podTopologyHints{}, policy: policy, - podMap: make(map[string]string), + podMap: containermap.NewContainerMap(), }, } } diff --git a/pkg/kubelet/cm/topologymanager/scope_test.go b/pkg/kubelet/cm/topologymanager/scope_test.go index 4d6315adb72c3..bb0dccf0a90ed 100644 --- a/pkg/kubelet/cm/topologymanager/scope_test.go +++ b/pkg/kubelet/cm/topologymanager/scope_test.go @@ -19,6 +19,7 @@ package topologymanager import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "reflect" "testing" ) @@ -64,14 +65,13 @@ func TestAddContainer(t *testing.T) { }, } scope := scope{} - scope.podMap = make(map[string]string) + scope.podMap = containermap.NewContainerMap() for _, tc := range testCases { pod := v1.Pod{} pod.UID = tc.podUID - err := scope.AddContainer(&pod, tc.containerID) - if err != nil { - t.Errorf("Expected error to be nil but got: %v", err) - } + container := v1.Container{} + container.Name = tc.name + scope.AddContainer(&pod, &container, tc.containerID) if val, ok := scope.podMap[tc.containerID]; ok { if reflect.DeepEqual(val, pod.UID) { t.Errorf("Error occurred") @@ -100,18 +100,27 @@ func TestRemoveContainer(t *testing.T) { }, } var len1, len2 int + var lenHints1, lenHints2 int scope := scope{} - scope.podMap = make(map[string]string) + scope.podMap = containermap.NewContainerMap() + scope.podTopologyHints = podTopologyHints{} for _, tc := range testCases { - scope.podMap[tc.containerID] = string(tc.podUID) + scope.podMap.Add(string(tc.podUID), tc.name, tc.containerID) + scope.podTopologyHints[string(tc.podUID)] = make(map[string]TopologyHint) + scope.podTopologyHints[string(tc.podUID)][tc.name] = TopologyHint{} len1 = len(scope.podMap) + lenHints1 = len(scope.podTopologyHints) err := scope.RemoveContainer(tc.containerID) len2 = len(scope.podMap) + lenHints2 = len(scope.podTopologyHints) if err != nil { t.Errorf("Expected error to be nil but got: %v", err) } if len1-len2 != 1 { - t.Errorf("Remove Pod resulted in error") + t.Errorf("Remove Pod from podMap resulted in error") + } + if lenHints1-lenHints2 != 1 { + t.Error("Remove Pod from podTopologyHints resulted in error") } } diff --git a/pkg/kubelet/cm/topologymanager/topology_manager.go b/pkg/kubelet/cm/topologymanager/topology_manager.go index f1e435260dece..4f327e6efc042 100644 --- a/pkg/kubelet/cm/topologymanager/topology_manager.go +++ b/pkg/kubelet/cm/topologymanager/topology_manager.go @@ -46,7 +46,7 @@ type Manager interface { // wants to be consulted with when making topology hints AddHintProvider(HintProvider) // AddContainer adds pod to Manager for tracking - AddContainer(pod *v1.Pod, containerID string) error + AddContainer(pod *v1.Pod, container *v1.Container, containerID string) // RemoveContainer removes pod from Manager tracking RemoveContainer(containerID string) error // Store is the interface for storing pod topology hints @@ -175,8 +175,8 @@ func (m *manager) AddHintProvider(h HintProvider) { m.scope.AddHintProvider(h) } -func (m *manager) AddContainer(pod *v1.Pod, containerID string) error { - return m.scope.AddContainer(pod, containerID) +func (m *manager) AddContainer(pod *v1.Pod, container *v1.Container, containerID string) { + m.scope.AddContainer(pod, container, containerID) } func (m *manager) RemoveContainer(containerID string) error { diff --git a/pkg/proxy/topology.go b/pkg/proxy/topology.go index 377cc65184cff..89223072361fe 100644 --- a/pkg/proxy/topology.go +++ b/pkg/proxy/topology.go @@ -81,12 +81,12 @@ func filterEndpointsWithHints(endpoints []Endpoint, hintsAnnotation string, node } } - if len(filteredEndpoints) > 0 { + if len(filteredEndpoints) == 0 { klog.Warningf("Skipping topology aware endpoint filtering since no hints were provided for zone %s", zone) - return filteredEndpoints + return endpoints } - return endpoints + return filteredEndpoints } // deprecatedTopologyFilter returns the appropriate endpoints based on the diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index 382d8023d736e..49a987b95a4af 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -354,7 +354,7 @@ func newSourceVIP(hns HostNetworkService, network string, ip string, mac string, func (ep *endpointsInfo) Cleanup() { Log(ep, "Endpoint Cleanup", 3) - if ep.refCount != nil { + if !ep.GetIsLocal() && ep.refCount != nil { *ep.refCount-- // Remove the remote hns endpoint, if no service is referring it @@ -1157,10 +1157,10 @@ func (proxier *Proxier) syncProxyRules() { } else { // We only share the refCounts for remote endpoints ep.refCount = proxier.endPointsRefCount.getRefCount(newHnsEndpoint.hnsID) + *ep.refCount++ } ep.hnsID = newHnsEndpoint.hnsID - *ep.refCount++ Log(ep, "Endpoint resource found", 3) } diff --git a/pkg/scheduler/framework/fake/listers.go b/pkg/scheduler/framework/fake/listers.go index 0352bf3821783..1b4482fa3d249 100644 --- a/pkg/scheduler/framework/fake/listers.go +++ b/pkg/scheduler/framework/fake/listers.go @@ -22,6 +22,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" appslisters "k8s.io/client-go/listers/apps/v1" @@ -311,7 +312,12 @@ func (classes StorageClassLister) Get(name string) (*storagev1.StorageClass, err return &sc, nil } } - return nil, fmt.Errorf("unable to find storage class: %s", name) + return nil, &errors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + Message: fmt.Sprintf("unable to find storage class: %s", name), + }, + } } // List lists all StorageClass in the indexer. diff --git a/pkg/scheduler/framework/plugins/volumezone/volume_zone.go b/pkg/scheduler/framework/plugins/volumezone/volume_zone.go index 91fb62e9aac33..2e91ffa39b179 100644 --- a/pkg/scheduler/framework/plugins/volumezone/volume_zone.go +++ b/pkg/scheduler/framework/plugins/volumezone/volume_zone.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" corelisters "k8s.io/client-go/listers/core/v1" @@ -108,11 +109,11 @@ func (pl *VolumeZone) Filter(ctx context.Context, _ *framework.CycleState, pod * } pvcName := volume.PersistentVolumeClaim.ClaimName if pvcName == "" { - return framework.NewStatus(framework.Error, "PersistentVolumeClaim had no name") + return framework.NewStatus(framework.UnschedulableAndUnresolvable, "PersistentVolumeClaim had no name") } pvc, err := pl.pvcLister.PersistentVolumeClaims(pod.Namespace).Get(pvcName) - if err != nil { - return framework.AsStatus(err) + if s := getErrorAsStatus(err); !s.IsSuccess() { + return s } if pvc == nil { @@ -123,28 +124,27 @@ func (pl *VolumeZone) Filter(ctx context.Context, _ *framework.CycleState, pod * if pvName == "" { scName := storagehelpers.GetPersistentVolumeClaimClass(pvc) if len(scName) == 0 { - return framework.NewStatus(framework.Error, fmt.Sprint("PersistentVolumeClaim had no pv name and storageClass name")) + return framework.NewStatus(framework.UnschedulableAndUnresolvable, "PersistentVolumeClaim had no pv name and storageClass name") } - class, _ := pl.scLister.Get(scName) - if class == nil { - return framework.NewStatus(framework.Error, fmt.Sprintf("StorageClass %q claimed by PersistentVolumeClaim %q not found", scName, pvcName)) - + class, err := pl.scLister.Get(scName) + if s := getErrorAsStatus(err); !s.IsSuccess() { + return s } if class.VolumeBindingMode == nil { - return framework.NewStatus(framework.Error, fmt.Sprintf("VolumeBindingMode not set for StorageClass %q", scName)) + return framework.NewStatus(framework.UnschedulableAndUnresolvable, fmt.Sprintf("VolumeBindingMode not set for StorageClass %q", scName)) } if *class.VolumeBindingMode == storage.VolumeBindingWaitForFirstConsumer { // Skip unbound volumes continue } - return framework.NewStatus(framework.Error, fmt.Sprint("PersistentVolume had no name")) + return framework.NewStatus(framework.UnschedulableAndUnresolvable, "PersistentVolume had no name") } pv, err := pl.pvLister.Get(pvName) - if err != nil { - return framework.AsStatus(err) + if s := getErrorAsStatus(err); !s.IsSuccess() { + return s } if pv == nil { @@ -171,6 +171,16 @@ func (pl *VolumeZone) Filter(ctx context.Context, _ *framework.CycleState, pod * return nil } +func getErrorAsStatus(err error) *framework.Status { + if err != nil { + if errors.IsNotFound(err) { + return framework.NewStatus(framework.UnschedulableAndUnresolvable, err.Error()) + } + return framework.AsStatus(err) + } + return nil +} + // New initializes a new plugin and returns it. func New(_ runtime.Object, handle framework.Handle) (framework.Plugin, error) { informerFactory := handle.SharedInformerFactory() diff --git a/pkg/scheduler/framework/plugins/volumezone/volume_zone_test.go b/pkg/scheduler/framework/plugins/volumezone/volume_zone_test.go index 4477e6cb9c0b7..1ec592aed17a7 100644 --- a/pkg/scheduler/framework/plugins/volumezone/volume_zone_test.go +++ b/pkg/scheduler/framework/plugins/volumezone/volume_zone_test.go @@ -413,21 +413,21 @@ func TestWithBinding(t *testing.T) { name: "unbound volume empty storage class", Pod: createPodWithVolume("pod_1", "vol_1", "PVC_EmptySC"), Node: testNode, - wantStatus: framework.NewStatus(framework.Error, + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, "PersistentVolumeClaim had no pv name and storageClass name"), }, { name: "unbound volume no storage class", Pod: createPodWithVolume("pod_1", "vol_1", "PVC_NoSC"), Node: testNode, - wantStatus: framework.NewStatus(framework.Error, - "StorageClass \"Class_0\" claimed by PersistentVolumeClaim \"PVC_NoSC\" not found"), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, + "unable to find storage class: Class_0"), }, { name: "unbound volume immediate binding mode", Pod: createPodWithVolume("pod_1", "vol_1", "PVC_ImmediateSC"), Node: testNode, - wantStatus: framework.NewStatus(framework.Error, "VolumeBindingMode not set for StorageClass \"Class_Immediate\""), + wantStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, "VolumeBindingMode not set for StorageClass \"Class_Immediate\""), }, { name: "unbound volume wait binding mode", diff --git a/pkg/scheduler/internal/queue/scheduling_queue.go b/pkg/scheduler/internal/queue/scheduling_queue.go index 393d11ff392b7..4b61d3271e672 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue.go +++ b/pkg/scheduler/internal/queue/scheduling_queue.go @@ -758,6 +758,8 @@ func newUnschedulablePodsMap(metricRecorder metrics.MetricRecorder) *Unschedulab // may be different than what scheduler has here. We should be able to find pods // by their UID and update/delete them. type nominatedPodMap struct { + // podLister is used to verify if the given pod is alive. + podLister listersv1.PodLister // nominatedPods is a map keyed by a node name and the value is a list of // pods which are nominated to run on the node. These are pods which can be in // the activeQ or unschedulableQ. @@ -781,6 +783,15 @@ func (npm *nominatedPodMap) add(pi *framework.PodInfo, nodeName string) { return } } + + if npm.podLister != nil { + // If the pod is not alive, don't contain it. + if _, err := npm.podLister.Pods(pi.Pod.Namespace).Get(pi.Pod.Name); err != nil { + klog.V(4).InfoS("Pod doesn't exist in podLister, aborting adding it to the nominated map", "pod", klog.KObj(pi.Pod)) + return + } + } + npm.nominatedPodToNode[pi.Pod.UID] = nnn for _, npi := range npm.nominatedPods[nnn] { if npi.Pod.UID == pi.Pod.UID { @@ -833,8 +844,17 @@ func (npm *nominatedPodMap) UpdateNominatedPod(oldPod *v1.Pod, newPodInfo *frame } // NewPodNominator creates a nominatedPodMap as a backing of framework.PodNominator. +// DEPRECATED: use NewSafePodNominator() instead. func NewPodNominator() framework.PodNominator { + return NewSafePodNominator(nil) +} + +// NewSafePodNominator creates a nominatedPodMap as a backing of framework.PodNominator. +// Unlike NewPodNominator, it passes in a podLister so as to check if the pod is alive +// before adding its nominatedNode info. +func NewSafePodNominator(podLister listersv1.PodLister) framework.PodNominator { return &nominatedPodMap{ + podLister: podLister, nominatedPods: make(map[string][]*framework.PodInfo), nominatedPodToNode: make(map[ktypes.UID]string), } diff --git a/pkg/scheduler/internal/queue/scheduling_queue_test.go b/pkg/scheduler/internal/queue/scheduling_queue_test.go index ab1adb80533ab..95a402e2d2084 100644 --- a/pkg/scheduler/internal/queue/scheduling_queue_test.go +++ b/pkg/scheduler/internal/queue/scheduling_queue_test.go @@ -25,12 +25,17 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ktypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" "k8s.io/component-base/metrics/testutil" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/scheduler/framework" @@ -137,8 +142,8 @@ func TestPriorityQueue_Add(t *testing.T) { "node1": {medPriorityPodInfo, unschedulablePodInfo}, }, } - if !reflect.DeepEqual(q.PodNominator, expectedNominatedPods) { - t.Errorf("Unexpected nominated map after adding pods. Expected: %v, got: %v", expectedNominatedPods, q.PodNominator) + if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominatedPodMap{}), cmpopts.IgnoreFields(nominatedPodMap{}, "RWMutex")); diff != "" { + t.Errorf("Unexpected diff after adding pods (-want, +got):\n%s", diff) } if p, err := q.Pop(); err != nil || p.Pod != highPriorityPodInfo.Pod { t.Errorf("Expected: %v after Pop, but got: %v", highPriorityPodInfo.Pod.Name, p.Pod.Name) @@ -189,8 +194,8 @@ func TestPriorityQueue_AddUnschedulableIfNotPresent(t *testing.T) { "node1": {highPriNominatedPodInfo, unschedulablePodInfo}, }, } - if !reflect.DeepEqual(q.PodNominator, expectedNominatedPods) { - t.Errorf("Unexpected nominated map after adding pods. Expected: %v, got: %v", expectedNominatedPods, q.PodNominator) + if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominatedPodMap{}), cmpopts.IgnoreFields(nominatedPodMap{}, "RWMutex")); diff != "" { + t.Errorf("Unexpected diff after adding pods (-want, +got):\n%s", diff) } if p, err := q.Pop(); err != nil || p.Pod != highPriNominatedPodInfo.Pod { t.Errorf("Expected: %v after Pop, but got: %v", highPriNominatedPodInfo.Pod.Name, p.Pod.Name) @@ -622,6 +627,57 @@ func TestPriorityQueue_NominatedPodsForNode(t *testing.T) { } } +func TestPriorityQueue_NominatedPodDeleted(t *testing.T) { + tests := []struct { + name string + podInfo *framework.PodInfo + deletePod bool + want bool + }{ + { + name: "alive pod gets added into PodNominator", + podInfo: medPriorityPodInfo, + want: true, + }, + { + name: "deleted pod shouldn't be added into PodNominator", + podInfo: highPriNominatedPodInfo, + deletePod: true, + want: false, + }, + { + name: "pod without .status.nominatedPodName specified shouldn't be added into PodNominator", + podInfo: highPriorityPodInfo, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cs := fake.NewSimpleClientset(tt.podInfo.Pod) + informerFactory := informers.NewSharedInformerFactory(cs, 0) + podLister := informerFactory.Core().V1().Pods().Lister() + + // Build a PriorityQueue. + q := NewPriorityQueue(newDefaultQueueSort(), informerFactory, WithPodNominator(NewSafePodNominator(podLister))) + ctx := context.Background() + informerFactory.Start(ctx.Done()) + informerFactory.WaitForCacheSync(ctx.Done()) + + if tt.deletePod { + // Simulate that the test pod gets deleted physically. + informerFactory.Core().V1().Pods().Informer().GetStore().Delete(tt.podInfo.Pod) + } + + q.AddNominatedPod(tt.podInfo, tt.podInfo.Pod.Status.NominatedNodeName) + + if got := len(q.NominatedPodsForNode(tt.podInfo.Pod.Status.NominatedNodeName)) == 1; got != tt.want { + t.Errorf("Want %v, but got %v", tt.want, got) + } + }) + } +} + func TestPriorityQueue_PendingPods(t *testing.T) { makeSet := func(pods []*v1.Pod) map[*v1.Pod]struct{} { pendingSet := map[*v1.Pod]struct{}{} @@ -669,15 +725,15 @@ func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { "node5": {unschedulablePodInfo}, }, } - if !reflect.DeepEqual(q.PodNominator, expectedNominatedPods) { - t.Errorf("Unexpected nominated map after adding pods. Expected: %v, got: %v", expectedNominatedPods, q.PodNominator) + if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominatedPodMap{}), cmpopts.IgnoreFields(nominatedPodMap{}, "RWMutex")); diff != "" { + t.Errorf("Unexpected diff after adding pods (-want, +got):\n%s", diff) } if p, err := q.Pop(); err != nil || p.Pod != medPriorityPodInfo.Pod { t.Errorf("Expected: %v after Pop, but got: %v", medPriorityPodInfo.Pod.Name, p.Pod.Name) } // List of nominated pods shouldn't change after popping them from the queue. - if !reflect.DeepEqual(q.PodNominator, expectedNominatedPods) { - t.Errorf("Unexpected nominated map after popping pods. Expected: %v, got: %v", expectedNominatedPods, q.PodNominator) + if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominatedPodMap{}), cmpopts.IgnoreFields(nominatedPodMap{}, "RWMutex")); diff != "" { + t.Errorf("Unexpected diff after popping pods (-want, +got):\n%s", diff) } // Update one of the nominated pods that doesn't have nominatedNodeName in the // pod object. It should be updated correctly. @@ -694,8 +750,8 @@ func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { "node5": {unschedulablePodInfo}, }, } - if !reflect.DeepEqual(q.PodNominator, expectedNominatedPods) { - t.Errorf("Unexpected nominated map after updating pods. Expected: %v, got: %v", expectedNominatedPods, q.PodNominator) + if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominatedPodMap{}), cmpopts.IgnoreFields(nominatedPodMap{}, "RWMutex")); diff != "" { + t.Errorf("Unexpected diff after updating pods (-want, +got):\n%s", diff) } // Delete a nominated pod that doesn't have nominatedNodeName in the pod @@ -711,8 +767,8 @@ func TestPriorityQueue_UpdateNominatedPodForNode(t *testing.T) { "node5": {unschedulablePodInfo}, }, } - if !reflect.DeepEqual(q.PodNominator, expectedNominatedPods) { - t.Errorf("Unexpected nominated map after deleting pods. Expected: %v, got: %v", expectedNominatedPods, q.PodNominator) + if diff := cmp.Diff(q.PodNominator, expectedNominatedPods, cmp.AllowUnexported(nominatedPodMap{}), cmpopts.IgnoreFields(nominatedPodMap{}, "RWMutex")); diff != "" { + t.Errorf("Unexpected diff after deleting pods (-want, +got):\n%s", diff) } } diff --git a/staging/publishing/rules.yaml b/staging/publishing/rules.yaml index bce3ca69965d0..7afa2dddb8671 100644 --- a/staging/publishing/rules.yaml +++ b/staging/publishing/rules.yaml @@ -7,7 +7,7 @@ recursive-delete-patterns: - BUILD.bazel - "*/BUILD.bazel" - Gopkg.toml -default-go-version: 1.16.4 +default-go-version: 1.16.5 rules: - destination: code-generator branches: diff --git a/staging/src/k8s.io/api/go.sum b/staging/src/k8s.io/api/go.sum index a98bf73c085b7..b90342ad8e7ee 100644 --- a/staging/src/k8s.io/api/go.sum +++ b/staging/src/k8s.io/api/go.sum @@ -80,7 +80,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -250,7 +249,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -932,7 +930,7 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY= diff --git a/staging/src/k8s.io/apiextensions-apiserver/go.sum b/staging/src/k8s.io/apiextensions-apiserver/go.sum index d29f1dc85cd99..55d45c67d7095 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/go.sum +++ b/staging/src/k8s.io/apiextensions-apiserver/go.sum @@ -89,7 +89,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -273,7 +272,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -1007,8 +1005,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 h1:0jaDAAxtqIrrqas4vtTqxct4xS5kHfRNycTRLTyJmVM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY= diff --git a/staging/src/k8s.io/apiserver/go.mod b/staging/src/k8s.io/apiserver/go.mod index 4c1c2c35ae0f4..78dc30e43277d 100644 --- a/staging/src/k8s.io/apiserver/go.mod +++ b/staging/src/k8s.io/apiserver/go.mod @@ -20,7 +20,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/hashicorp/golang-lru v0.5.1 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 - github.com/openshift/library-go v0.0.0-20210521084623-7392ea9b02ca + github.com/openshift/library-go v0.0.0-20210611143017-0d0ef669a361 github.com/pkg/errors v0.9.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.6.1 @@ -42,7 +42,7 @@ require ( k8s.io/klog/v2 v2.8.0 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 k8s.io/utils v0.0.0-20210521133846-da695404a2bc - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 sigs.k8s.io/structured-merge-diff/v4 v4.1.0 sigs.k8s.io/yaml v1.2.0 ) diff --git a/staging/src/k8s.io/apiserver/go.sum b/staging/src/k8s.io/apiserver/go.sum index e88e5dfede488..c994f5b22f8e5 100644 --- a/staging/src/k8s.io/apiserver/go.sum +++ b/staging/src/k8s.io/apiserver/go.sum @@ -88,7 +88,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -273,7 +272,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -1003,8 +1001,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 h1:0jaDAAxtqIrrqas4vtTqxct4xS5kHfRNycTRLTyJmVM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY= diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater.go index 91e2e9691473e..ad651a2b59db2 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater.go @@ -25,6 +25,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +const totalAnnotationSizeLimitB int64 = 256 * (1 << 10) // 256 kB + type lastAppliedUpdater struct { fieldManager Manager } @@ -78,8 +80,8 @@ func hasLastApplied(obj runtime.Object) bool { if annotations == nil { return false } - _, ok := annotations[corev1.LastAppliedConfigAnnotation] - return ok + lastApplied, ok := annotations[corev1.LastAppliedConfigAnnotation] + return ok && len(lastApplied) > 0 } func setLastApplied(obj runtime.Object, value string) error { @@ -92,6 +94,9 @@ func setLastApplied(obj runtime.Object, value string) error { annotations = map[string]string{} } annotations[corev1.LastAppliedConfigAnnotation] = value + if isAnnotationsValid(annotations) != nil { + delete(annotations, corev1.LastAppliedConfigAnnotation) + } accessor.SetAnnotations(annotations) return nil } @@ -115,3 +120,14 @@ func buildLastApplied(obj runtime.Object) (string, error) { } return string(lastApplied), nil } + +func isAnnotationsValid(annotations map[string]string) error { + var totalSize int64 + for k, v := range annotations { + totalSize += (int64)(len(k)) + (int64)(len(v)) + } + if totalSize > (int64)(totalAnnotationSizeLimitB) { + return fmt.Errorf("annotations size %d is larger than limit %d", totalSize, totalAnnotationSizeLimitB) + } + return nil +} diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater_test.go index 14e9aa167da06..ff467c8074e0b 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater_test.go @@ -17,9 +17,13 @@ limitations under the License. package fieldmanager import ( + "fmt" "strings" "testing" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/yaml" @@ -89,3 +93,134 @@ spec: t.Errorf("expected last applied annotation to be updated, but got: %q", lastApplied) } } + +func TestLargeLastApplied(t *testing.T) { + tests := []struct { + name string + oldObject *corev1.ConfigMap + newObject *corev1.ConfigMap + }{ + { + name: "old object + new object last-applied annotation is too big", + oldObject: &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "large-update-test-cm", + Namespace: "default", + Annotations: map[string]string{ + corev1.LastAppliedConfigAnnotation: "nonempty", + }, + }, + Data: map[string]string{"k": "v"}, + }, + newObject: func() *corev1.ConfigMap { + cfg := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "large-update-test-cm", + Namespace: "default", + Annotations: map[string]string{ + corev1.LastAppliedConfigAnnotation: "nonempty", + }, + }, + Data: map[string]string{"k": "v"}, + } + for i := 0; i < 9999; i++ { + unique := fmt.Sprintf("this-key-is-very-long-so-as-to-create-a-very-large-serialized-fieldset-%v", i) + cfg.Data[unique] = "A" + } + return cfg + }(), + }, + { + name: "old object + new object annotations + new object last-applied annotation is too big", + oldObject: func() *corev1.ConfigMap { + cfg := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "large-update-test-cm", + Namespace: "default", + Annotations: map[string]string{ + corev1.LastAppliedConfigAnnotation: "nonempty", + }, + }, + Data: map[string]string{"k": "v"}, + } + for i := 0; i < 2000; i++ { + unique := fmt.Sprintf("this-key-is-very-long-so-as-to-create-a-very-large-serialized-fieldset-%v", i) + cfg.Data[unique] = "A" + } + return cfg + }(), + newObject: func() *corev1.ConfigMap { + cfg := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "large-update-test-cm", + Namespace: "default", + Annotations: map[string]string{ + corev1.LastAppliedConfigAnnotation: "nonempty", + }, + }, + Data: map[string]string{"k": "v"}, + } + for i := 0; i < 2000; i++ { + unique := fmt.Sprintf("this-key-is-very-long-so-as-to-create-a-very-large-serialized-fieldset-%v", i) + cfg.Data[unique] = "A" + cfg.ObjectMeta.Annotations[unique] = "A" + } + return cfg + }(), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f := NewTestFieldManager(schema.FromAPIVersionAndKind("v1", "ConfigMap"), + false, + func(m Manager) Manager { + return NewLastAppliedUpdater(m) + }) + + if err := f.Apply(test.oldObject, "kubectl", false); err != nil { + t.Errorf("Error applying object: %v", err) + } + + lastApplied, err := getLastApplied(f.liveObj) + if err != nil { + t.Errorf("Failed to access last applied annotation: %v", err) + } + if len(lastApplied) == 0 || lastApplied == "nonempty" { + t.Errorf("Expected an updated last-applied annotation, but got: %q", lastApplied) + } + + if err := f.Apply(test.newObject, "kubectl", false); err != nil { + t.Errorf("Error applying object: %v", err) + } + + accessor := meta.NewAccessor() + annotations, err := accessor.Annotations(f.liveObj) + if err != nil { + t.Errorf("Failed to access annotations: %v", err) + } + if annotations == nil { + t.Errorf("No annotations on obj: %v", f.liveObj) + } + lastApplied, ok := annotations[corev1.LastAppliedConfigAnnotation] + if ok || len(lastApplied) > 0 { + t.Errorf("Expected no last applied annotation, but got last applied with length: %d", len(lastApplied)) + } + }) + } +} diff --git a/staging/src/k8s.io/cli-runtime/go.sum b/staging/src/k8s.io/cli-runtime/go.sum index 3db2a9aa39915..535628bb10105 100644 --- a/staging/src/k8s.io/cli-runtime/go.sum +++ b/staging/src/k8s.io/cli-runtime/go.sum @@ -82,7 +82,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -258,7 +257,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -964,7 +962,7 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.8 h1:G2z6JPSSjtWWgMeWSoHdXqyftJNmMmyxXpwENGoOtGE= diff --git a/staging/src/k8s.io/client-go/tools/cache/reflector.go b/staging/src/k8s.io/client-go/tools/cache/reflector.go index c732c782207b6..90fa45ddbfeb1 100644 --- a/staging/src/k8s.io/client-go/tools/cache/reflector.go +++ b/staging/src/k8s.io/client-go/tools/cache/reflector.go @@ -216,13 +216,13 @@ var internalPackages = []string{"client-go/tools/cache/"} // objects and subsequent deltas. // Run will exit when stopCh is closed. func (r *Reflector) Run(stopCh <-chan struct{}) { - klog.V(2).Infof("Starting reflector %s (%s) from %s", r.expectedTypeName, r.resyncPeriod, r.name) + klog.V(3).Infof("Starting reflector %s (%s) from %s", r.expectedTypeName, r.resyncPeriod, r.name) wait.BackoffUntil(func() { if err := r.ListAndWatch(stopCh); err != nil { r.watchErrorHandler(r, err) } }, r.backoffManager, true, stopCh) - klog.V(2).Infof("Stopping reflector %s (%s) from %s", r.expectedTypeName, r.resyncPeriod, r.name) + klog.V(3).Infof("Stopping reflector %s (%s) from %s", r.expectedTypeName, r.resyncPeriod, r.name) } var ( diff --git a/staging/src/k8s.io/cloud-provider/go.sum b/staging/src/k8s.io/cloud-provider/go.sum index 3069fdbe605e3..6f6cfd567a047 100644 --- a/staging/src/k8s.io/cloud-provider/go.sum +++ b/staging/src/k8s.io/cloud-provider/go.sum @@ -89,7 +89,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -274,7 +273,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -1007,8 +1005,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 h1:0jaDAAxtqIrrqas4vtTqxct4xS5kHfRNycTRLTyJmVM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY= diff --git a/staging/src/k8s.io/code-generator/examples/go.sum b/staging/src/k8s.io/code-generator/examples/go.sum index 0ab881ad12604..cc4665146adcb 100644 --- a/staging/src/k8s.io/code-generator/examples/go.sum +++ b/staging/src/k8s.io/code-generator/examples/go.sum @@ -104,8 +104,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -290,8 +290,8 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= diff --git a/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go b/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go index 3b31c80a7a038..517351a3b4dac 100644 --- a/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go +++ b/staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go @@ -95,7 +95,7 @@ func (c *controller) Run(stopCh <-chan struct{}) { klog.Infof("lease controller has nil lease client, will not claim or renew leases") return } - wait.Until(c.sync, c.renewInterval, stopCh) + wait.JitterUntil(c.sync, c.renewInterval, 0.04, true, stopCh) } func (c *controller) sync() { diff --git a/staging/src/k8s.io/controller-manager/go.sum b/staging/src/k8s.io/controller-manager/go.sum index 3bad2ab807f69..ad934906b9850 100644 --- a/staging/src/k8s.io/controller-manager/go.sum +++ b/staging/src/k8s.io/controller-manager/go.sum @@ -111,7 +111,6 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -338,7 +337,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -1220,8 +1218,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 h1:0jaDAAxtqIrrqas4vtTqxct4xS5kHfRNycTRLTyJmVM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY= diff --git a/staging/src/k8s.io/kube-aggregator/go.sum b/staging/src/k8s.io/kube-aggregator/go.sum index 48f6c21080dcb..4c1ecbc136c20 100644 --- a/staging/src/k8s.io/kube-aggregator/go.sum +++ b/staging/src/k8s.io/kube-aggregator/go.sum @@ -89,7 +89,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -274,7 +273,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -1010,8 +1008,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 h1:0jaDAAxtqIrrqas4vtTqxct4xS5kHfRNycTRLTyJmVM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY= diff --git a/staging/src/k8s.io/kube-controller-manager/go.sum b/staging/src/k8s.io/kube-controller-manager/go.sum index 9752177eb6dc9..a4297ba75720b 100644 --- a/staging/src/k8s.io/kube-controller-manager/go.sum +++ b/staging/src/k8s.io/kube-controller-manager/go.sum @@ -651,7 +651,7 @@ k8s.io/utils v0.0.0-20210521133846-da695404a2bc/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/staging/src/k8s.io/kubectl/go.sum b/staging/src/k8s.io/kubectl/go.sum index 7a5883dcec762..b3380c5ac2acf 100644 --- a/staging/src/k8s.io/kubectl/go.sum +++ b/staging/src/k8s.io/kubectl/go.sum @@ -85,7 +85,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -270,7 +269,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -993,7 +991,7 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.8 h1:G2z6JPSSjtWWgMeWSoHdXqyftJNmMmyxXpwENGoOtGE= diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer.go index 0fa0115804583..3d59abc8879a3 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer.go @@ -65,6 +65,10 @@ var ( defaultNlbHealthCheckThreshold = int64(3) defaultHealthCheckPort = "traffic-port" defaultHealthCheckPath = "/" + + // Defaults for ELB Target operations + defaultRegisterTargetsChunkSize = 100 + defaultDeregisterTargetsChunkSize = 100 ) func isNLB(annotations map[string]string) bool { @@ -563,6 +567,7 @@ func (c *Cloud) deleteListenerV2(listener *elbv2.Listener) error { // ensureTargetGroup creates a target group with a set of instances. func (c *Cloud) ensureTargetGroup(targetGroup *elbv2.TargetGroup, serviceName types.NamespacedName, mapping nlbPortMapping, instances []string, vpcID string, tags map[string]string) (*elbv2.TargetGroup, error) { dirty := false + expectedTargets := c.computeTargetGroupExpectedTargets(instances, mapping.TrafficPort) if targetGroup == nil { targetType := "instance" name := c.buildTargetGroupName(serviceName, mapping.FrontendPort, mapping.TrafficPort, mapping.TrafficProtocol, targetType, mapping) @@ -609,86 +614,23 @@ func (c *Cloud) ensureTargetGroup(targetGroup *elbv2.TargetGroup, serviceName ty } } - registerInput := &elbv2.RegisterTargetsInput{ - TargetGroupArn: result.TargetGroups[0].TargetGroupArn, - Targets: []*elbv2.TargetDescription{}, - } - for _, instanceID := range instances { - registerInput.Targets = append(registerInput.Targets, &elbv2.TargetDescription{ - Id: aws.String(string(instanceID)), - Port: aws.Int64(mapping.TrafficPort), - }) - } - - _, err = c.elbv2.RegisterTargets(registerInput) - if err != nil { - return nil, fmt.Errorf("error registering targets for load balancer: %q", err) + tg := result.TargetGroups[0] + tgARN := aws.StringValue(tg.TargetGroupArn) + if err := c.ensureTargetGroupTargets(tgARN, expectedTargets, nil); err != nil { + return nil, err } - - return result.TargetGroups[0], nil + return tg, nil } // handle instances in service { - healthResponse, err := c.elbv2.DescribeTargetHealth(&elbv2.DescribeTargetHealthInput{TargetGroupArn: targetGroup.TargetGroupArn}) + tgARN := aws.StringValue(targetGroup.TargetGroupArn) + actualTargets, err := c.obtainTargetGroupActualTargets(tgARN) if err != nil { - return nil, fmt.Errorf("error describing target group health: %q", err) - } - actualIDs := []string{} - for _, healthDescription := range healthResponse.TargetHealthDescriptions { - if aws.StringValue(healthDescription.TargetHealth.State) == elbv2.TargetHealthStateEnumHealthy { - actualIDs = append(actualIDs, *healthDescription.Target.Id) - } else if healthDescription.TargetHealth.Reason != nil { - switch aws.StringValue(healthDescription.TargetHealth.Reason) { - case elbv2.TargetHealthReasonEnumTargetDeregistrationInProgress: - // We don't need to count this instance in service if it is - // on its way out - default: - actualIDs = append(actualIDs, *healthDescription.Target.Id) - } - } - } - - actual := sets.NewString(actualIDs...) - expected := sets.NewString(instances...) - - additions := expected.Difference(actual) - removals := actual.Difference(expected) - - if len(additions) > 0 { - registerInput := &elbv2.RegisterTargetsInput{ - TargetGroupArn: targetGroup.TargetGroupArn, - Targets: []*elbv2.TargetDescription{}, - } - for instanceID := range additions { - registerInput.Targets = append(registerInput.Targets, &elbv2.TargetDescription{ - Id: aws.String(instanceID), - Port: aws.Int64(mapping.TrafficPort), - }) - } - _, err := c.elbv2.RegisterTargets(registerInput) - if err != nil { - return nil, fmt.Errorf("error registering new targets in target group: %q", err) - } - dirty = true + return nil, err } - - if len(removals) > 0 { - deregisterInput := &elbv2.DeregisterTargetsInput{ - TargetGroupArn: targetGroup.TargetGroupArn, - Targets: []*elbv2.TargetDescription{}, - } - for instanceID := range removals { - deregisterInput.Targets = append(deregisterInput.Targets, &elbv2.TargetDescription{ - Id: aws.String(instanceID), - Port: aws.Int64(mapping.TrafficPort), - }) - } - _, err := c.elbv2.DeregisterTargets(deregisterInput) - if err != nil { - return nil, fmt.Errorf("error trying to deregister targets in target group: %q", err) - } - dirty = true + if err := c.ensureTargetGroupTargets(tgARN, expectedTargets, actualTargets); err != nil { + return nil, err } } @@ -738,6 +680,101 @@ func (c *Cloud) ensureTargetGroup(targetGroup *elbv2.TargetGroup, serviceName ty return targetGroup, nil } +func (c *Cloud) ensureTargetGroupTargets(tgARN string, expectedTargets []*elbv2.TargetDescription, actualTargets []*elbv2.TargetDescription) error { + targetsToRegister, targetsToDeregister := c.diffTargetGroupTargets(expectedTargets, actualTargets) + if len(targetsToRegister) > 0 { + targetsToRegisterChunks := c.chunkTargetDescriptions(targetsToRegister, defaultRegisterTargetsChunkSize) + for _, targetsChunk := range targetsToRegisterChunks { + req := &elbv2.RegisterTargetsInput{ + TargetGroupArn: aws.String(tgARN), + Targets: targetsChunk, + } + if _, err := c.elbv2.RegisterTargets(req); err != nil { + return fmt.Errorf("error trying to register targets in target group: %q", err) + } + } + } + if len(targetsToDeregister) > 0 { + targetsToDeregisterChunks := c.chunkTargetDescriptions(targetsToDeregister, defaultDeregisterTargetsChunkSize) + for _, targetsChunk := range targetsToDeregisterChunks { + req := &elbv2.DeregisterTargetsInput{ + TargetGroupArn: aws.String(tgARN), + Targets: targetsChunk, + } + if _, err := c.elbv2.DeregisterTargets(req); err != nil { + return fmt.Errorf("error trying to deregister targets in target group: %q", err) + } + } + } + return nil +} + +func (c *Cloud) computeTargetGroupExpectedTargets(instanceIDs []string, port int64) []*elbv2.TargetDescription { + expectedTargets := make([]*elbv2.TargetDescription, 0, len(instanceIDs)) + for _, instanceID := range instanceIDs { + expectedTargets = append(expectedTargets, &elbv2.TargetDescription{ + Id: aws.String(instanceID), + Port: aws.Int64(port), + }) + } + return expectedTargets +} + +func (c *Cloud) obtainTargetGroupActualTargets(tgARN string) ([]*elbv2.TargetDescription, error) { + req := &elbv2.DescribeTargetHealthInput{ + TargetGroupArn: aws.String(tgARN), + } + resp, err := c.elbv2.DescribeTargetHealth(req) + if err != nil { + return nil, fmt.Errorf("error describing target group health: %q", err) + } + actualTargets := make([]*elbv2.TargetDescription, 0, len(resp.TargetHealthDescriptions)) + for _, targetDesc := range resp.TargetHealthDescriptions { + if targetDesc.TargetHealth.Reason != nil && aws.StringValue(targetDesc.TargetHealth.Reason) == elbv2.TargetHealthReasonEnumTargetDeregistrationInProgress { + continue + } + actualTargets = append(actualTargets, targetDesc.Target) + } + return actualTargets, nil +} + +// diffTargetGroupTargets computes the targets to register and targets to deregister based on existingTargets and desired instances. +func (c *Cloud) diffTargetGroupTargets(expectedTargets []*elbv2.TargetDescription, actualTargets []*elbv2.TargetDescription) (targetsToRegister []*elbv2.TargetDescription, targetsToDeregister []*elbv2.TargetDescription) { + expectedTargetsByUID := make(map[string]*elbv2.TargetDescription, len(expectedTargets)) + for _, target := range expectedTargets { + targetUID := fmt.Sprintf("%v:%v", aws.StringValue(target.Id), aws.Int64Value(target.Port)) + expectedTargetsByUID[targetUID] = target + } + actualTargetsByUID := make(map[string]*elbv2.TargetDescription, len(actualTargets)) + for _, target := range actualTargets { + targetUID := fmt.Sprintf("%v:%v", aws.StringValue(target.Id), aws.Int64Value(target.Port)) + actualTargetsByUID[targetUID] = target + } + + expectedTargetsUIDs := sets.StringKeySet(expectedTargetsByUID) + actualTargetsUIDs := sets.StringKeySet(actualTargetsByUID) + for _, targetUID := range expectedTargetsUIDs.Difference(actualTargetsUIDs).List() { + targetsToRegister = append(targetsToRegister, expectedTargetsByUID[targetUID]) + } + for _, targetUID := range actualTargetsUIDs.Difference(expectedTargetsUIDs).List() { + targetsToDeregister = append(targetsToDeregister, actualTargetsByUID[targetUID]) + } + return targetsToRegister, targetsToDeregister +} + +// chunkTargetDescriptions will split slice of TargetDescription into chunks +func (c *Cloud) chunkTargetDescriptions(targets []*elbv2.TargetDescription, chunkSize int) [][]*elbv2.TargetDescription { + var chunks [][]*elbv2.TargetDescription + for i := 0; i < len(targets); i += chunkSize { + end := i + chunkSize + if end > len(targets) { + end = len(targets) + } + chunks = append(chunks, targets[i:end]) + } + return chunks +} + // updateInstanceSecurityGroupsForNLB will adjust securityGroup's settings to allow inbound traffic into instances from clientCIDRs and portMappings. // TIP: if either instances or clientCIDRs or portMappings are nil, then the securityGroup rules for lbName are cleared. func (c *Cloud) updateInstanceSecurityGroupsForNLB(lbName string, instances map[InstanceID]*ec2.Instance, subnetCIDRs []string, clientCIDRs []string, portMappings []nlbPortMapping) error { diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer_test.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer_test.go index 5ec1d1b3219f4..15bc8eec0bac6 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer_test.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/elbv2" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" @@ -540,3 +541,455 @@ func TestFilterTargetNodes(t *testing.T) { }) } } + +func TestCloud_chunkTargetDescriptions(t *testing.T) { + type args struct { + targets []*elbv2.TargetDescription + chunkSize int + } + tests := []struct { + name string + args args + want [][]*elbv2.TargetDescription + }{ + { + name: "can be evenly chunked", + args: args{ + targets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdefg1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg3"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg4"), + Port: aws.Int64(8080), + }, + }, + chunkSize: 2, + }, + want: [][]*elbv2.TargetDescription{ + { + { + Id: aws.String("i-abcdefg1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg2"), + Port: aws.Int64(8080), + }, + }, + { + { + Id: aws.String("i-abcdefg3"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg4"), + Port: aws.Int64(8080), + }, + }, + }, + }, + { + name: "cannot be evenly chunked", + args: args{ + targets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdefg1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg3"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg4"), + Port: aws.Int64(8080), + }, + }, + chunkSize: 3, + }, + want: [][]*elbv2.TargetDescription{ + { + { + Id: aws.String("i-abcdefg1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg3"), + Port: aws.Int64(8080), + }, + }, + { + + { + Id: aws.String("i-abcdefg4"), + Port: aws.Int64(8080), + }, + }, + }, + }, + { + name: "chunkSize equal to total count", + args: args{ + targets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdefg1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg3"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg4"), + Port: aws.Int64(8080), + }, + }, + chunkSize: 4, + }, + want: [][]*elbv2.TargetDescription{ + { + { + Id: aws.String("i-abcdefg1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg3"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg4"), + Port: aws.Int64(8080), + }, + }, + }, + }, + { + name: "chunkSize greater than total count", + args: args{ + targets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdefg1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg3"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg4"), + Port: aws.Int64(8080), + }, + }, + chunkSize: 10, + }, + want: [][]*elbv2.TargetDescription{ + { + { + Id: aws.String("i-abcdefg1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg3"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdefg4"), + Port: aws.Int64(8080), + }, + }, + }, + }, + { + name: "chunk nil slice", + args: args{ + targets: nil, + chunkSize: 2, + }, + want: nil, + }, + { + name: "chunk empty slice", + args: args{ + targets: []*elbv2.TargetDescription{}, + chunkSize: 2, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Cloud{} + got := c.chunkTargetDescriptions(tt.args.targets, tt.args.chunkSize) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestCloud_diffTargetGroupTargets(t *testing.T) { + type args struct { + expectedTargets []*elbv2.TargetDescription + actualTargets []*elbv2.TargetDescription + } + tests := []struct { + name string + args args + wantTargetsToRegister []*elbv2.TargetDescription + wantTargetsToDeregister []*elbv2.TargetDescription + }{ + { + name: "all targets to register", + args: args{ + expectedTargets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef2"), + Port: aws.Int64(8080), + }, + }, + actualTargets: nil, + }, + wantTargetsToRegister: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef2"), + Port: aws.Int64(8080), + }, + }, + wantTargetsToDeregister: nil, + }, + { + name: "all targets to deregister", + args: args{ + expectedTargets: nil, + actualTargets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef2"), + Port: aws.Int64(8080), + }, + }, + }, + wantTargetsToRegister: nil, + wantTargetsToDeregister: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef2"), + Port: aws.Int64(8080), + }, + }, + }, + { + name: "some targets to register and deregister", + args: args{ + expectedTargets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef4"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef5"), + Port: aws.Int64(8080), + }, + }, + actualTargets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef3"), + Port: aws.Int64(8080), + }, + }, + }, + wantTargetsToRegister: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef4"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef5"), + Port: aws.Int64(8080), + }, + }, + wantTargetsToDeregister: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef3"), + Port: aws.Int64(8080), + }, + }, + }, + { + name: "both expected and actual targets are empty", + args: args{ + expectedTargets: nil, + actualTargets: nil, + }, + wantTargetsToRegister: nil, + wantTargetsToDeregister: nil, + }, + { + name: "expected and actual targets equals", + args: args{ + expectedTargets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef3"), + Port: aws.Int64(8080), + }, + }, + actualTargets: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef3"), + Port: aws.Int64(8080), + }, + }, + }, + wantTargetsToRegister: nil, + wantTargetsToDeregister: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Cloud{} + gotTargetsToRegister, gotTargetsToDeregister := c.diffTargetGroupTargets(tt.args.expectedTargets, tt.args.actualTargets) + assert.Equal(t, tt.wantTargetsToRegister, gotTargetsToRegister) + assert.Equal(t, tt.wantTargetsToDeregister, gotTargetsToDeregister) + }) + } +} + +func TestCloud_computeTargetGroupExpectedTargets(t *testing.T) { + type args struct { + instanceIDs []string + port int64 + } + tests := []struct { + name string + args args + want []*elbv2.TargetDescription + }{ + { + name: "no instance", + args: args{ + instanceIDs: nil, + port: 8080, + }, + want: []*elbv2.TargetDescription{}, + }, + { + name: "one instance", + args: args{ + instanceIDs: []string{"i-abcdef1"}, + port: 8080, + }, + want: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + }, + }, + { + name: "multiple instances", + args: args{ + instanceIDs: []string{"i-abcdef1", "i-abcdef2", "i-abcdef3"}, + port: 8080, + }, + want: []*elbv2.TargetDescription{ + { + Id: aws.String("i-abcdef1"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef2"), + Port: aws.Int64(8080), + }, + { + Id: aws.String("i-abcdef3"), + Port: aws.Int64(8080), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Cloud{} + got := c.computeTargetGroupExpectedTargets(tt.args.instanceIDs, tt.args.port) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go index 2c1c4dfaa3f26..72bbd3632ff00 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_loadbalancer.go @@ -2238,9 +2238,11 @@ func (az *Cloud) reconcilePublicIP(clusterName string, service *v1.Service, lbNa } dirtyPIP = true } - changed := az.ensurePIPTagged(service, &pip) - if changed { - dirtyPIP = true + if !isUserAssignedPIP { + changed := az.ensurePIPTagged(service, &pip) + if changed { + dirtyPIP = true + } } if shouldReleaseExistingOwnedPublicIP(&pip, wantLb, isInternal, isUserAssignedPIP, desiredPipName, serviceIPTagRequest) { // Then, release the public ip diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go index 6f9fdb6dc9313..f6a146fc5c612 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_standard.go @@ -365,9 +365,13 @@ func (az *Cloud) serviceOwnsFrontendIP(fip network.FrontendIPConfiguration, serv klog.Warningf("serviceOwnsFrontendIP: unexpected error when finding match public IP of the service %s with loadBalancerLP %s: %v", service.Name, loadBalancerIP, err) return false, isPrimaryService, nil } - - if pip != nil && pip.ID != nil && pip.PublicIPAddressPropertiesFormat != nil && pip.IPAddress != nil { - if strings.EqualFold(*pip.ID, *fip.PublicIPAddress.ID) { + if pip != nil && + pip.ID != nil && + pip.PublicIPAddressPropertiesFormat != nil && + pip.IPAddress != nil && + fip.FrontendIPConfigurationPropertiesFormat != nil && + fip.FrontendIPConfigurationPropertiesFormat.PublicIPAddress != nil { + if strings.EqualFold(to.String(pip.ID), to.String(fip.PublicIPAddress.ID)) { klog.V(4).Infof("serviceOwnsFrontendIP: found secondary service %s of the frontend IP config %s", service.Name, *fip.Name) return true, isPrimaryService, nil diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go index 8504aada304b8..351a1b9eeef54 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss.go @@ -1033,7 +1033,6 @@ func (ss *scaleSet) EnsureHostInPool(service *v1.Service, nodeName types.NodeNam }) primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools newVM := &compute.VirtualMachineScaleSetVM{ - Sku: vm.Sku, Location: vm.Location, VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ HardwareProfile: vm.HardwareProfile, @@ -1175,7 +1174,6 @@ func (ss *scaleSet) ensureVMSSInPool(service *v1.Service, nodes []*v1.Node, back }) primaryIPConfig.LoadBalancerBackendAddressPools = &loadBalancerBackendAddressPools newVMSS := compute.VirtualMachineScaleSet{ - Sku: vmss.Sku, Location: vmss.Location, VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ @@ -1356,7 +1354,6 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromNode(nodeName, backendPoolID str // Compose a new vmssVM with added backendPoolID. primaryIPConfiguration.LoadBalancerBackendAddressPools = &newBackendPools newVM := &compute.VirtualMachineScaleSetVM{ - Sku: vm.Sku, Location: vm.Location, VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ HardwareProfile: vm.HardwareProfile, @@ -1478,7 +1475,6 @@ func (ss *scaleSet) ensureBackendPoolDeletedFromVMSS(service *v1.Service, backen // Compose a new vmss with added backendPoolID. primaryIPConfig.LoadBalancerBackendAddressPools = &newBackendPools newVMSS := compute.VirtualMachineScaleSet{ - Sku: vmss.Sku, Location: vmss.Location, VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go index 5abfec1936e91..9dcf785f5ad9a 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_cache.go @@ -193,6 +193,11 @@ func (ss *scaleSet) newVMSSVirtualMachinesCache(resourceGroupName, vmssName, cac } computerName := strings.ToLower(*vm.OsProfile.ComputerName) + if vm.NetworkProfile == nil || vm.NetworkProfile.NetworkInterfaces == nil { + klog.Warningf("skip caching vmssVM %s since its network profile hasn't initialized yet (probably still under creating)", computerName) + continue + } + vmssVMCacheEntry := &vmssVirtualMachinesEntry{ resourceGroup: resourceGroupName, vmssName: vmssName, diff --git a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go index 3d5a00d7e79ed..9cc91ea4bad8b 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/azure/azure_vmss_test.go @@ -1385,16 +1385,25 @@ func TestGetAgentPoolScaleSets(t *testing.T) { { VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000000")}, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, + }, }, }, { VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000001")}, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, + }, }, }, { VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000002")}, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, + }, }, }, } @@ -1463,6 +1472,20 @@ func TestGetVMSetNames(t *testing.T) { }, expectedErr: fmt.Errorf("scale set (vmss-1) - not found"), }, + { + description: "GetVMSetNames should report an error if vm's network profile is nil", + service: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ServiceAnnotationLoadBalancerMode: "vmss"}}, + }, + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "vmss-vm-000003", + }, + }, + }, + expectedErr: fmt.Errorf("instance not found"), + }, { description: "GetVMSetNames should return the correct vmss names", service: &v1.Service{ @@ -1496,16 +1519,30 @@ func TestGetVMSetNames(t *testing.T) { { VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000000")}, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, + }, }, }, { VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000001")}, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, + }, }, }, { VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000002")}, + NetworkProfile: &compute.NetworkProfile{ + NetworkInterfaces: &[]compute.NetworkInterfaceReference{}, + }, + }, + }, + { + VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ + OsProfile: &compute.OSProfile{ComputerName: to.StringPtr("vmss-vm-000003")}, }, }, } @@ -1732,7 +1769,6 @@ func TestEnsureHostInPool(t *testing.T) { expectedVMSSName: testVMSSName, expectedInstanceID: "0", expectedVMSSVM: &compute.VirtualMachineScaleSetVM{ - Sku: &compute.Sku{Name: to.StringPtr("sku")}, Location: to.StringPtr("westus"), VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ NetworkProfileConfiguration: &compute.VirtualMachineScaleSetVMNetworkProfileConfiguration{ @@ -2129,7 +2165,6 @@ func TestEnsureBackendPoolDeletedFromNode(t *testing.T) { expectedVMSSName: testVMSSName, expectedInstanceID: "0", expectedVMSSVM: &compute.VirtualMachineScaleSetVM{ - Sku: &compute.Sku{Name: to.StringPtr("sku")}, Location: to.StringPtr("westus"), VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ NetworkProfileConfiguration: &compute.VirtualMachineScaleSetVMNetworkProfileConfiguration{ diff --git a/staging/src/k8s.io/legacy-cloud-providers/go.sum b/staging/src/k8s.io/legacy-cloud-providers/go.sum index 398b31a3c2df8..f3a9af93e75d9 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/go.sum +++ b/staging/src/k8s.io/legacy-cloud-providers/go.sum @@ -734,7 +734,7 @@ k8s.io/utils v0.0.0-20210521133846-da695404a2bc/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/staging/src/k8s.io/sample-apiserver/go.sum b/staging/src/k8s.io/sample-apiserver/go.sum index 120d54dcf1ca0..c94611b1f812a 100644 --- a/staging/src/k8s.io/sample-apiserver/go.sum +++ b/staging/src/k8s.io/sample-apiserver/go.sum @@ -730,8 +730,8 @@ k8s.io/utils v0.0.0-20210521133846-da695404a2bc/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 h1:0jaDAAxtqIrrqas4vtTqxct4xS5kHfRNycTRLTyJmVM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/test/e2e/generated/bindata.go b/test/e2e/generated/bindata.go index 1c1c4dcead3ad..57ce557c51d03 100644 --- a/test/e2e/generated/bindata.go +++ b/test/e2e/generated/bindata.go @@ -1737,7 +1737,7 @@ func testE2eTestingManifestsSampleDevicePluginYaml() (*asset, error) { return a, nil } -var _testE2eTestingManifestsSchedulingNvidiaDriverInstallerYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x54\x4d\x6f\xe3\x36\x10\xbd\xeb\x57\x0c\xb2\x67\x59\xc9\xb6\x45\x17\xba\x19\x71\xd0\x1a\xcd\xda\x41\xec\xe4\x6a\xd0\xe4\x58\x22\x44\x91\xea\x70\xa8\x8d\xff\x7d\x41\xc9\xf1\x4a\xde\x38\xf1\x76\x75\xd3\x7c\xbc\xf9\xe0\x7b\xf3\x09\xd6\xa5\xf6\x30\x13\x58\x3b\xbb\x42\x86\x6f\xc2\x83\x23\x5d\x68\x2b\x8c\xd9\x03\xe1\x0e\x09\xad\x44\x05\x3b\x72\x75\xf2\x09\x4a\xe6\xc6\xe7\x59\x56\x68\x2e\xc3\x76\x22\x5d\x9d\xfd\xe5\x5c\x61\xf0\xd6\xb8\xa0\x1e\x8c\xe0\x9d\xa3\x3a\x93\xce\xb2\xd0\x16\x29\x45\x5b\x68\x8b\xa9\x90\x12\x0d\x92\x60\x47\x3e\xdb\x1a\xb7\xcd\x6a\xe1\x19\x29\x53\x5d\x6d\x8f\x3c\xd9\x8b\xda\x24\x49\x6c\x09\x61\xe6\x64\x85\xb4\xd3\x06\x41\x58\x05\x8e\x4b\x24\xf0\x2e\x90\x44\xd8\x39\x02\x8e\x6d\x1f\x53\x41\x10\x82\xb6\x3f\xd3\x9e\x4f\x8b\x26\xa4\xda\x7a\x16\xc6\x20\x25\x89\x68\xf4\x33\x92\xd7\xce\xe6\x20\x9a\xc6\x67\xed\x4d\x52\x69\xab\xf2\xef\xdb\x49\x6a\x64\xa1\x04\x8b\x3c\x01\xb0\xa2\xc6\x1c\x6c\xab\x95\x16\xa9\x22\xdd\x22\x0d\xe0\x7a\xbf\x6f\x84\xc4\x1c\xaa\xb0\xc5\xd4\xef\x3d\x63\x9d\x00\x18\xb1\x45\xe3\x23\x04\x40\xf5\xc5\xa7\xa2\x69\xce\xe3\xf8\x06\x65\x0c\xf5\x68\x50\xb2\xa3\x3e\xad\x16\x2c\xcb\xfb\x01\xce\x05\x48\x00\xa1\x51\x82\x71\xc5\x24\x18\x8b\x7d\x9f\xc8\xfb\x06\x73\x78\x74\xc6\x68\x5b\x3c\x75\x01\x09\x00\x63\xdd\x18\xc1\x78\xa8\x36\x98\x3a\x7e\x66\x54\xf8\xe3\x4d\x5c\xdc\x20\xc0\xeb\xb8\xf1\x13\xbb\x9d\xb6\x9a\xf7\x83\x42\x4e\xe1\xf4\x07\x2b\x00\xe1\xbf\x41\x13\xaa\x59\x20\x6d\x8b\x95\x2c\x51\x85\x38\xcf\xbc\xb0\xee\x68\xbe\x7b\x41\x19\x38\x3e\xef\x20\xb3\xc7\x5c\x1d\x96\xbb\x46\xaa\xfd\xd8\x9d\xf6\xbb\xbe\x7b\x69\x08\x7d\x24\xc7\x89\x3f\x46\x54\xb8\xcf\x41\x46\x7e\x4d\x8a\x8e\x6b\x1d\xed\x8a\x6a\x44\xfa\x93\x2c\x00\xd7\xf4\x8e\x1c\xee\x5e\xb4\x67\x7f\x08\x60\xd7\x25\x0c\x2b\xa5\x83\xd8\xab\x3e\xf8\xea\xe0\x2a\x9d\xe7\x05\xf2\x37\x47\x55\x0e\x4c\x01\x07\xf6\x87\xf9\x6c\x64\x6b\x9d\x09\x35\x0e\x50\xfb\x77\x53\xd8\x1e\x7b\xeb\xd2\x04\x97\xc3\x19\x9b\xf8\x0f\xd9\xf7\xb0\xd7\xc4\x36\x98\x4a\xd8\x54\x4b\x95\xd6\x2e\x58\xbe\x04\xa5\x74\x35\x66\x51\x0e\x64\x91\xd1\x67\x5b\x6d\xb3\x9e\x0f\x59\x0f\x97\x69\xa9\x26\xea\xa4\xd2\x81\x31\x07\xaa\xa4\x4a\x53\x1a\x6b\xfc\x5a\xc1\x93\x1a\xe4\x1c\x5f\x3e\xc7\xc1\x12\x99\x78\xfb\x7a\xe9\x06\x9b\xd5\xb5\x28\x30\x87\x42\xd2\x44\xbb\xee\xd8\x74\xf4\xf8\xf1\xec\xe4\xed\xe7\xeb\xcf\xd7\xd7\x7f\x5e\xdf\xfc\xa4\x9a\x08\xfb\x63\xe8\x4f\x75\x80\x9e\x4f\x28\x2a\x9b\x90\xc3\xf5\xe4\xe6\x8f\xa3\xd5\xa3\x0c\xa4\x79\x1f\x5b\xc7\x17\x1e\xcd\x47\xba\xd5\x06\x0b\x54\x23\xee\x00\xa0\x6d\x87\x71\xaf\x6b\x5b\x3c\xcf\x67\xf3\xe9\x66\xbe\x58\xad\xa7\xf7\xf7\x9b\xd9\xfc\x71\xf3\xf7\x72\xb5\x1e\x35\xd0\x0a\x13\xf0\x92\xb7\xf8\x00\xf8\x76\xb9\x58\x4f\xe7\x8b\xbb\xc7\x37\xd1\x83\xa7\xcc\x38\x29\xcc\x79\xcc\xe7\xa7\xfb\x7f\xa6\x8b\xcd\xfc\x76\xf6\x7f\x1b\x7d\x8b\xa5\xef\x54\x78\xbf\x63\x64\xf9\x11\xde\xe3\x72\xb9\xde\x7c\x5d\x3e\x2d\xd6\x11\xef\x4d\x94\x48\xdc\xa3\xa3\xd7\xf8\xd7\x48\xe3\x01\x0b\x2e\xd5\x11\x40\x27\x80\x87\x9e\xe3\x67\x17\xfa\xe1\x01\x18\xe3\x9c\x1d\xf3\xad\x0b\x34\x4e\x1d\x7a\xde\x11\xea\x38\x69\xb0\x10\x79\x5e\x9b\x57\xd5\x17\x3f\x39\x08\xb4\x11\xc1\x63\xfe\xdb\xe4\xf7\xc9\xcd\xd5\x89\x0e\x3b\x57\x92\xfc\x17\x00\x00\xff\xff\x76\x9a\x5b\x51\x28\x09\x00\x00") +var _testE2eTestingManifestsSchedulingNvidiaDriverInstallerYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x55\x4b\x6f\xe3\x36\x10\xbe\xeb\x57\x0c\xb2\x97\x16\x85\x24\x3f\x92\x6c\xa2\x53\x83\x38\xd8\x1a\x4d\xec\x20\x76\xf6\x6a\x50\xe4\x48\x26\x4c\x91\x2a\x39\xd2\xc6\xfd\xf5\x05\x25\xc5\x91\x9d\x97\xb7\xeb\x9b\x39\x33\xdf\x3c\xf4\xcd\x37\x5f\x60\xb9\x96\x0e\x26\x0c\x0b\xa3\x17\x48\xf0\x83\x39\x30\x56\xe6\x52\x33\xa5\xb6\x60\x31\x43\x8b\x9a\xa3\x80\xcc\x9a\x22\xf8\x02\x6b\xa2\xd2\x25\x71\x9c\x4b\x5a\x57\x69\xc4\x4d\x11\x7f\x33\x26\x57\x78\xad\x4c\x25\xee\x15\xa3\xcc\xd8\x22\xe6\x46\x13\x93\x1a\x6d\x88\x3a\x97\x1a\x43\xc6\x39\x2a\xb4\x8c\x8c\x75\x71\xaa\x4c\x1a\x17\xcc\x11\xda\x58\x34\xb9\x1d\x52\xb4\x65\x85\x0a\x02\x5f\x12\xc2\xc4\xf0\x0d\xda\x4c\x2a\x04\xa6\x05\x18\x5a\xa3\x05\x67\x2a\xcb\x11\x32\x63\x81\x7c\xd9\xbb\x50\x60\x16\x41\xea\x9f\x29\xcf\x85\x79\x59\x85\x52\x3b\x62\x4a\xa1\x0d\x02\x56\xca\xef\x68\x9d\x34\x3a\x01\x56\x96\x2e\xae\x87\xc1\x46\x6a\x91\xbc\x4c\x27\x28\x90\x98\x60\xc4\x92\x00\x40\xb3\x02\x13\xd0\xb5\x14\x92\x85\xc2\xca\x1a\x6d\x0f\xae\xb5\xbb\x92\x71\x4c\x60\x53\xa5\x18\xba\xad\x23\x2c\x02\x00\xc5\x52\x54\xce\x43\x00\x6c\x2e\x5c\xc8\xca\xf2\x7d\x1c\x57\x22\xf7\xae\x0e\x15\x72\x32\xb6\x0d\x2b\x18\xf1\xf5\x6d\x0f\xe7\x08\x24\x80\xaa\x14\x8c\x70\x41\x96\x11\xe6\xdb\x36\x90\xb6\x25\x26\xf0\x60\x94\x92\x3a\x7f\x6c\x1c\x02\x00\xc2\xa2\x54\x8c\xb0\xcb\xd6\xeb\xda\xff\xd4\x5e\xe2\xcf\x27\x71\x74\x81\x00\xcf\xed\xfa\x1f\xcb\x32\xa9\x25\x6d\x7b\x89\x8c\xc0\xab\x57\xaf\x00\x16\xff\xa9\xa4\x45\x31\xa9\xac\xd4\xf9\x82\xaf\x51\x54\xbe\x9f\x69\xae\xcd\xee\xf9\xe6\x09\x79\x45\xfe\xf3\xf6\x22\x5b\xcc\x45\x37\xdc\x25\xda\xc2\xed\x9b\xc3\x76\xd6\x37\x4f\xa5\x45\xe7\xc9\x71\x60\xf7\x1e\x1b\xdc\x26\xc0\x3d\xbf\xa2\xbc\xe1\x5a\x43\xbb\x7c\xb3\x47\xfa\x83\x28\x00\x53\xb6\x86\x04\x6e\x9e\xa4\x23\xd7\x39\x90\x69\x02\xfa\x99\xc2\x9e\xef\x49\xeb\x7c\xd2\x99\xd6\xc6\xd1\x0c\xe9\x87\xb1\x9b\x04\xc8\x56\xd8\x7b\xbf\x9f\x4e\xf6\xde\x6a\xa3\xaa\x02\x7b\xa8\xed\x77\x13\x58\xef\x6a\x6b\xc2\x18\xad\xfb\x3d\x96\xfe\x3f\xc4\x2f\x6e\xcf\x81\x75\xa5\x36\x4c\x87\x92\x8b\xb0\x30\x95\xa6\x63\x50\xd6\xa6\xc0\xd8\xaf\x83\xd5\x48\xe8\xe2\x54\xea\xb8\xe5\x43\xdc\xc2\xc5\x92\x8b\x48\x1c\x64\xea\x18\xd3\x51\x25\x14\xd2\x86\x3e\xc7\xaf\x25\x3c\xc8\x61\x8d\xa1\xe3\xfb\xe8\x5e\x3c\x13\xaf\x9f\x95\xae\xc7\x8c\x56\xc3\xae\xe7\x0b\xf8\x76\xff\x08\x3b\x8a\x83\x2c\x58\x8e\x50\xb7\x32\x03\x05\xdb\x42\x8a\x20\xb0\x44\x2d\x50\x13\x18\x0d\xb4\x7e\xb1\x9b\xac\x81\x48\x51\xea\x1c\x2a\x87\x22\xea\x25\x78\xf0\xca\x0c\x64\x40\x20\x31\xa9\x1c\xb0\xd4\x54\xd4\xc4\xf7\xf2\xe9\x9d\x20\x72\xe3\x3a\x76\xb6\x2a\xda\x70\x94\x1b\x17\x93\x31\xca\xc5\x7f\xc4\x16\x33\x17\xaf\x91\x09\xf7\xac\xcc\xce\xf2\x98\x17\xc2\x7b\xad\xf2\xb2\x5a\xed\x70\xe3\x5e\x1d\x5e\x9d\xa9\x6b\xd6\xa2\x42\xe6\x10\xb4\x21\x74\xf0\xdb\x2e\xf5\xe1\x6a\xbc\x1c\x07\x53\x92\x2c\xe4\xbf\x28\x42\xe3\x62\x61\xb8\x8b\x3b\x8c\xb0\xc1\xf8\xbd\x6b\x10\x6d\x21\xf5\xcb\x60\x5e\x0f\xd6\x9f\x04\x06\xb9\xac\x51\x1f\x8c\x2f\x0a\x7a\xc5\xde\xb1\xd2\x79\xc8\x9c\xdb\x48\x9a\xe6\x0a\x34\xc5\xbd\xbe\x07\x49\x3d\x8a\x06\xd1\x18\x42\x70\x95\x24\x96\xaa\xf6\xec\xf8\xcc\x77\x17\x67\xc0\x1c\x94\x68\xe1\xd7\x3b\xfc\xe2\x33\x5f\x9c\x85\xc3\xf1\x78\x38\x08\x87\xa3\xc1\x65\x38\xde\x31\xb3\xa1\x4b\x72\x44\xb5\x7f\xba\x35\x1b\x9d\x9d\x27\x43\x9e\x8d\xbe\x0e\x86\x82\x8f\xf8\xf8\xf2\xf4\x94\x5d\x8e\x33\x31\x38\xe7\xe9\x39\xbf\x44\x14\x19\x4b\xb3\xf1\x68\x7c\x3a\x3a\x3b\xbd\x18\xa7\x6c\xcc\xb2\xd1\xe5\xe9\xd9\x70\x70\x3e\x1a\xa6\xe9\xf8\xab\x18\xe0\x4f\x4a\xba\xed\xb8\xe4\x0e\xc5\x18\x1d\x1d\xe8\x24\x2f\xab\x04\x06\xd1\xf0\x6c\xf7\xea\x90\x57\x56\xd2\xd6\xef\x0f\x3e\xd1\xde\x92\x59\x59\x4b\x85\x39\x8a\x3d\x01\x03\x40\x5d\xf7\xfd\x9e\x77\x77\xf6\x7d\x3a\x99\x5e\xad\xa6\xb3\xc5\xf2\xea\xf6\x76\x35\x99\x3e\xac\xfe\x9a\x2f\x96\x7b\x05\xd4\x4c\x55\x78\x8c\x20\x7c\x02\x7c\x3d\x9f\x2d\xaf\xa6\xb3\x9b\x87\x37\xd1\x2b\x67\x63\x65\x38\x53\xef\x63\x7e\x7f\xbc\xfd\xfb\x6a\xb6\x9a\x5e\x4f\xfe\x6f\xa1\x6f\x49\xe5\x07\x19\x3e\xae\x18\x89\x7f\x86\xf7\x30\x9f\x2f\x57\x77\xf3\xc7\xd9\xd2\xe3\xbd\x89\xe2\xd5\x73\x67\x68\x0f\xcd\x9d\xd7\xd2\x1e\x0b\x8e\x15\x73\x80\x46\x85\xef\x5b\xa1\x7d\x77\xa0\x9f\x5e\xa1\x7d\x9c\x77\xdb\x7c\xeb\x0c\xee\x87\xf6\x2d\x1f\x5c\x8b\xfd\xa0\xde\x40\xf8\xab\x03\xb1\xdb\xec\x93\xcd\x85\x8b\xba\xf5\x2e\x59\xe5\x30\x19\x47\xa7\xd1\xf0\xe4\x60\x0f\x1b\x53\x10\xfc\x17\x00\x00\xff\xff\x80\x97\xe4\x6a\xad\x0b\x00\x00") func testE2eTestingManifestsSchedulingNvidiaDriverInstallerYamlBytes() ([]byte, error) { return bindataRead( @@ -2877,7 +2877,7 @@ func testE2e_nodeTestingManifestsSriovdpSaYaml() (*asset, error) { return a, nil } -var _testImagesMakefile = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x53\x6f\x6f\xa3\x36\x1c\x7e\x7d\xfe\x14\x8f\xc2\xbd\x68\xa5\x40\xda\x68\xdb\x49\x99\xaa\x13\x4d\x59\x8b\xda\x83\x1b\xd0\xab\xfa\x2a\x72\xe0\x17\xb0\x4a\x6c\x66\x9b\x26\xd1\xb4\xef\x3e\x19\x9a\x6d\x59\x37\x1d\xef\x6c\x3f\xff\xfc\xf8\x87\x87\xa5\xea\x0e\x5a\xd4\x8d\xc5\xfc\xe2\xf2\x13\x8a\x86\x70\xdf\xaf\x49\x4b\xb2\x64\x10\xf6\xb6\x51\xda\x04\xcc\x63\x1e\x1e\x44\x49\xd2\x50\x85\x5e\x56\xa4\x61\x1b\x42\xd8\xf1\xb2\xa1\xe3\xc9\x14\xdf\x48\x1b\xa1\x24\xe6\xc1\x05\xce\x1c\x60\xf2\x76\x34\x39\xff\x99\x79\x38\xa8\x1e\x5b\x7e\x80\x54\x16\xbd\x21\xd8\x46\x18\x6c\x44\x4b\xa0\x7d\x49\x9d\x85\x90\x28\xd5\xb6\x6b\x05\x97\x25\x61\x27\x6c\x33\xd8\xbc\x89\x04\xcc\xc3\xf3\x9b\x84\x5a\x5b\x2e\x24\x38\x4a\xd5\x1d\xa0\x36\xff\xc4\x81\xdb\x21\xb0\xfb\x1a\x6b\xbb\xc5\x6c\xb6\xdb\xed\x02\x3e\x84\x0d\x94\xae\x67\xed\x08\x34\xb3\x87\x78\x19\x25\x79\xe4\xcf\x83\x8b\x81\xf2\x28\x5b\x32\x06\x9a\x7e\xeb\x85\xa6\x0a\xeb\x03\x78\xd7\xb5\xa2\xe4\xeb\x96\xd0\xf2\x1d\x94\x06\xaf\x35\x51\x05\xab\x5c\xde\x9d\x16\x56\xc8\x7a\x0a\xa3\x36\x76\xc7\x35\x31\x0f\x95\x30\x56\x8b\x75\x6f\x4f\xca\x3a\xa6\x13\xe6\x04\xa0\x24\xb8\xc4\x24\xcc\x11\xe7\x13\x5c\x87\x79\x9c\x4f\x99\x87\xa7\xb8\xb8\x4b\x1f\x0b\x3c\x85\x59\x16\x26\x45\x1c\xe5\x48\x33\x2c\xd3\xe4\x26\x2e\xe2\x34\xc9\x91\xfe\x82\x30\x79\xc6\x7d\x9c\xdc\x4c\x41\xc2\x36\xa4\x41\xfb\x4e\xbb\xfc\x4a\x43\xb8\x1a\xa9\x72\x9d\xe5\x44\x27\x01\x36\x6a\x0c\x64\x3a\x2a\xc5\x46\x94\x68\xb9\xac\x7b\x5e\x13\x6a\xf5\x4a\x5a\x0a\x59\xa3\x23\xbd\x15\xc6\x3d\xa6\x01\x97\x15\xf3\xd0\x8a\xad\xb0\xdc\x0e\x3b\xef\x2e\x15\x30\x96\x45\xb7\x71\x5e\x64\xcf\xf8\x7c\x85\xba\xd4\x81\x50\xb3\x97\xbf\x26\xc9\xa7\x39\xf9\x96\x8c\xf5\xc5\x96\xd7\x64\xd8\x6d\x1a\x66\x5f\x1c\xf4\x13\xbb\x49\x97\xf7\x51\xb6\x5a\x46\x59\xb1\xba\x0e\xf3\x68\xf5\x35\x2c\xee\xf0\xf9\x8a\xfd\x1a\x7d\x79\xfc\x16\x65\x79\x9c\x26\x57\xaf\x3f\x06\x97\xc1\x85\x3f\x67\xb7\xe9\x43\x98\xdc\xae\x8e\xfb\x97\xc1\xe5\x4f\xc1\x0f\x8c\xf6\x9d\xd2\x96\x31\xb1\x91\x15\x6d\xf0\x74\x17\x16\xec\xe3\x19\x69\xad\xf4\xb0\x70\xa5\xf3\xbf\x9f\xf5\x95\x6b\xe1\x5e\x74\x0a\xda\x2f\xb0\xe5\x2f\x04\xde\xb6\x03\xf2\x4a\x92\x3d\x67\x24\x2b\xb1\x61\xcc\xc3\x75\x2f\xda\x0a\xa5\xaa\x68\xfc\x09\x42\x5d\x9b\xc5\x30\x5b\x0e\xbc\xc0\x8d\xd0\x54\x5a\xa5\x0f\x90\x7c\x4b\xc6\x0d\xc5\xda\x51\x46\x74\xb4\xe7\xdb\xae\xa5\x91\x70\x6a\x53\xb6\xbd\xb1\xa4\x79\x27\x86\x5e\x48\x33\xde\xb6\x0b\x77\xee\x97\x4a\xba\xe1\x26\xcd\xd8\xc9\x72\xc1\x3e\x04\xb3\xa1\x3f\xbf\xb7\xa2\x0d\x4c\x33\x7a\xe1\xe3\x99\x93\x3c\xc7\xa4\x52\xe5\x0b\xe9\xc9\xc8\xeb\x7a\xd3\x2c\xd8\x87\x35\x37\x0d\xfc\x3d\xbe\x43\xd5\x54\xbb\xa1\x3c\x4c\xfe\x9f\xe1\x04\x8f\x84\xd1\x62\xd0\xf0\xb9\xac\x8e\x66\xff\x69\xb2\xe2\xb2\x5a\x8d\xe4\xdf\x1d\xf9\x0f\xc6\x82\xaf\x77\x69\xf2\x3c\x5c\x17\xef\x85\x70\x8c\xff\xaf\x3a\xfe\x0c\x00\x00\xff\xff\x35\xea\xd1\x8c\xaf\x04\x00\x00") +var _testImagesMakefile = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\x53\x6f\x6f\xa3\x36\x1c\x7e\x7d\xfe\x14\x8f\xc2\xbd\x68\xa5\x40\xda\x48\xb7\x93\x32\x55\x27\x9a\xb2\x16\xb5\x07\x37\xa0\x57\xf5\x55\xe4\xc0\x2f\x60\x95\xd8\xcc\x36\x4d\xa2\x69\xdf\x7d\x32\x34\xdb\xb2\x6e\x3a\xde\xd9\x7e\xfe\xf9\xf1\x0f\x0f\x4b\xd5\x1d\xb4\xa8\x1b\x8b\xf9\xc5\xe5\x67\x14\x0d\xe1\xbe\x5f\x93\x96\x64\xc9\x20\xec\x6d\xa3\xb4\x09\x98\xc7\x3c\x3c\x88\x92\xa4\xa1\x0a\xbd\xac\x48\xc3\x36\x84\xb0\xe3\x65\x43\xc7\x93\x29\xbe\x93\x36\x42\x49\xcc\x83\x0b\x9c\x39\xc0\xe4\xed\x68\x72\xfe\x33\xf3\x70\x50\x3d\xb6\xfc\x00\xa9\x2c\x7a\x43\xb0\x8d\x30\xd8\x88\x96\x40\xfb\x92\x3a\x0b\x21\x51\xaa\x6d\xd7\x0a\x2e\x4b\xc2\x4e\xd8\x66\xb0\x79\x13\x09\x98\x87\xe7\x37\x09\xb5\xb6\x5c\x48\x70\x94\xaa\x3b\x40\x6d\xfe\x89\x03\xb7\x43\x60\xf7\x35\xd6\x76\x8b\xd9\x6c\xb7\xdb\x05\x7c\x08\x1b\x28\x5d\xcf\xda\x11\x68\x66\x0f\xf1\x32\x4a\xf2\xc8\x9f\x07\x17\x03\xe5\x51\xb6\x64\x0c\x34\xfd\xd6\x0b\x4d\x15\xd6\x07\xf0\xae\x6b\x45\xc9\xd7\x2d\xa1\xe5\x3b\x28\x0d\x5e\x6b\xa2\x0a\x56\xb9\xbc\x3b\x2d\xac\x90\xf5\x14\x46\x6d\xec\x8e\x6b\x62\x1e\x2a\x61\xac\x16\xeb\xde\x9e\x94\x75\x4c\x27\xcc\x09\x40\x49\x70\x89\x49\x98\x23\xce\x27\xb8\x0e\xf3\x38\x9f\x32\x0f\x4f\x71\x71\x97\x3e\x16\x78\x0a\xb3\x2c\x4c\x8a\x38\xca\x91\x66\x58\xa6\xc9\x4d\x5c\xc4\x69\x92\x23\xfd\x05\x61\xf2\x8c\xfb\x38\xb9\x99\x82\x84\x6d\x48\x83\xf6\x9d\x76\xf9\x95\x86\x70\x35\x52\xe5\x3a\xcb\x89\x4e\x02\x6c\xd4\x18\xc8\x74\x54\x8a\x8d\x28\xd1\x72\x59\xf7\xbc\x26\xd4\xea\x95\xb4\x14\xb2\x46\x47\x7a\x2b\x8c\x7b\x4c\x03\x2e\x2b\xe6\xa1\x15\x5b\x61\xb9\x1d\x76\xde\x5d\x2a\x60\x2c\x8b\x6e\xe3\xbc\xc8\x9e\xf1\xe5\x0a\x75\xa9\x03\xa1\x66\x2f\x7f\x4d\x92\x4f\x73\xf2\x2d\x19\xeb\x8b\x2d\xaf\xc9\xb0\xdb\x34\xcc\xbe\x3a\xe8\x67\x76\x93\x2e\xef\xa3\x6c\xb5\x8c\xb2\x62\x75\x1d\xe6\xd1\xea\x5b\x58\xdc\xe1\xcb\x15\xfb\x35\xfa\xfa\xf8\x3d\xca\xf2\x38\x4d\xae\x5e\x3f\x05\x97\xc1\x85\x3f\x67\xb7\xe9\x43\x98\xdc\xae\x8e\xfb\x97\xc1\xe5\x4f\xc1\x27\x46\xfb\x4e\x69\xcb\x98\xd8\xc8\x8a\x36\x78\xba\x0b\x0b\xf6\xf1\x8c\xb4\x56\x7a\x58\xb8\xd2\xf9\xdf\xcf\xfa\xca\xb5\x70\x2f\x3a\x05\xed\x17\xd8\xf2\x17\x02\x6f\xdb\x01\x79\x25\xc9\x9e\x33\x92\x95\xd8\x30\xe6\xe1\xba\x17\x6d\x85\x52\x55\x34\xfe\x04\xa1\xae\xcd\x62\x98\x2d\x07\x5e\xe0\x46\x68\x2a\xad\xd2\x07\x48\xbe\x25\xe3\x86\x62\xed\x28\x23\x3a\xda\xf3\x6d\xd7\xd2\x48\x38\xb5\x29\xdb\xde\x58\xd2\xbc\x13\x43\x2f\xa4\x19\x6f\xdb\x85\x3b\xf7\x4b\x25\xdd\x70\x93\x66\xec\x64\xb9\x60\x1f\x82\xd9\xd0\x9f\xdf\x5b\xd1\x06\xa6\x19\xbd\xf0\xf1\xcc\x49\x9e\x63\x52\xa9\xf2\x85\xf4\x64\xe4\x75\xbd\x69\x16\xec\xc3\x9a\x9b\x06\xfe\x1e\x3f\xa0\x6a\xaa\xdd\x50\x1e\x26\xff\xcf\x70\x82\x47\xc2\x68\x31\x68\xf8\x5c\x56\x47\xb3\xff\x34\x59\x71\x59\xad\x46\xf2\xef\x8e\xfc\x07\x63\xc1\xb7\xbb\x34\x79\x1e\xae\x8b\xf7\x42\x38\xc6\xff\x57\x1d\x7f\x06\x00\x00\xff\xff\xe0\x9f\x84\x98\xaf\x04\x00\x00") func testImagesMakefileBytes() ([]byte, error) { return bindataRead( @@ -7177,7 +7177,7 @@ func testImagesSampleApiserverDockerfile_windows() (*asset, error) { return a, nil } -var _testImagesSampleApiserverMakefile = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x54\x61\x6f\xdb\x36\x10\xfd\x1c\xfe\x8a\x07\x2b\xe8\x9a\xc1\x92\xec\xa0\x68\x87\x14\xc6\xa6\xda\x9e\x63\x34\xb5\x06\xcb\x69\xd1\xa1\x40\x40\x49\x67\x89\x8b\x44\x6a\x24\x65\xd9\x08\xf2\xdf\x07\xca\x4e\xba\x22\x18\x30\x7f\x91\x25\xde\xbd\x7b\x7c\xf7\xee\x3c\x4c\x55\x73\xd0\xa2\x28\x2d\x2e\x47\x97\x23\x6c\x4a\xc2\xc7\x36\x25\x2d\xc9\x92\x41\xd4\xda\x52\x69\x13\x30\x8f\x79\xb8\x11\x19\x49\x43\x39\x5a\x99\x93\x86\x2d\x09\x51\xc3\xb3\x92\x9e\x4e\x86\xf8\x4c\xda\x08\x25\x71\x19\x8c\xf0\xda\x05\x0c\x4e\x47\x83\x8b\xf7\xcc\xc3\x41\xb5\xa8\xf9\x01\x52\x59\xb4\x86\x60\x4b\x61\xb0\x15\x15\x81\xf6\x19\x35\x16\x42\x22\x53\x75\x53\x09\x2e\x33\x42\x27\x6c\xd9\x97\x39\x81\x04\xcc\xc3\xd7\x13\x84\x4a\x2d\x17\x12\x1c\x99\x6a\x0e\x50\xdb\x7f\xc7\x81\xdb\x9e\xb0\xfb\x95\xd6\x36\x57\x61\xd8\x75\x5d\xc0\x7b\xb2\x81\xd2\x45\x58\x1d\x03\x4d\x78\xb3\x9c\xce\x57\xc9\xdc\xbf\x0c\x46\x7d\xca\xad\xac\xc8\x18\x68\xfa\xbb\x15\x9a\x72\xa4\x07\xf0\xa6\xa9\x44\xc6\xd3\x8a\x50\xf1\x0e\x4a\x83\x17\x9a\x28\x87\x55\x8e\x6f\xa7\x85\x15\xb2\x18\xc2\xa8\xad\xed\xb8\x26\xe6\x21\x17\xc6\x6a\x91\xb6\xf6\x07\xb1\x9e\xd8\x09\xf3\x43\x80\x92\xe0\x12\x83\x28\xc1\x32\x19\xe0\x43\x94\x2c\x93\x21\xf3\xf0\x65\xb9\xb9\x8e\x6f\x37\xf8\x12\xad\xd7\xd1\x6a\xb3\x9c\x27\x88\xd7\x98\xc6\xab\xd9\x72\xb3\x8c\x57\x09\xe2\xdf\x11\xad\xbe\xe2\xe3\x72\x35\x1b\x82\x84\x2d\x49\x83\xf6\x8d\x76\xfc\x95\x86\x70\x32\x52\xee\x34\x4b\x88\x7e\x20\xb0\x55\x47\x42\xa6\xa1\x4c\x6c\x45\x86\x8a\xcb\xa2\xe5\x05\xa1\x50\x3b\xd2\x52\xc8\x02\x0d\xe9\x5a\x18\xd7\x4c\x03\x2e\x73\xe6\xa1\x12\xb5\xb0\xdc\xf6\x5f\x5e\x5c\x2a\x60\x2c\x4e\xf0\xeb\x04\x95\x90\xed\x9e\x45\xeb\xe9\xb5\x7b\xe3\x75\xfe\xf6\x0d\xdb\x44\xeb\xc5\x7c\xe3\xde\xcf\x5f\x4f\x6f\xd7\xb3\xe5\xfa\x82\x2d\xe2\x68\xfd\x09\x13\xbc\x63\x8b\xf8\x26\x5a\x2d\xee\x3e\xcf\xd7\xc9\x32\x5e\xf5\x18\xdc\x92\xb1\x2c\x59\x4f\xef\x66\xcb\x35\x5c\x9a\x54\x36\x17\x1a\xe7\xaf\x4d\x49\x55\x85\xa6\xcb\x2f\x2e\x18\xed\x1b\xa5\x2d\x63\x1e\x3e\xb4\xa2\xca\xb1\x1b\x07\xe3\x77\xc1\xc8\x35\x86\xa4\x69\xf5\xf1\xd6\x59\xab\x35\x49\x0b\x4d\x15\x71\x43\x30\x6d\xe3\xd2\x0c\x38\x1a\x2d\x94\xc6\xee\x64\xda\x93\x89\x0c\xaf\x9b\x8a\xc0\x1b\x61\x48\xef\x48\x33\x0f\x0b\xb2\xbd\x19\x55\x6b\x91\xba\x52\x4e\x21\xab\xd0\xa8\xa6\x75\x64\x51\xab\xbc\xad\x08\x99\x33\x18\xf3\xdc\x14\xc9\x21\x8a\x53\x16\xe2\x24\x74\x82\xf8\xcf\x7a\x93\xdc\xb9\xf4\x1e\x8a\xa5\x42\x5e\xb1\xb3\x5c\x65\xf7\xa4\xa1\x5b\x09\xdf\xd7\x35\x7c\x01\x7f\x87\xc1\xf9\xc3\x51\xbc\xc7\xab\xef\xff\xfe\x1c\xe0\xfe\x17\x13\x14\x99\x0e\x84\x0a\x7b\x10\x5f\xd4\xbc\xa0\xf0\xbe\x4d\xc9\xcf\xb4\x32\xe6\xca\x69\xf1\x36\x78\xe3\x8f\xf1\x8d\x9d\x9d\x85\xa9\x90\x61\xca\x4d\x09\x3f\xc3\xc0\x7d\x39\xab\xef\x9d\xa0\x7e\x83\xb0\x50\xa1\xd1\x59\xff\x4c\x85\xc4\xab\x57\x7d\xca\xd9\x22\x1e\x8f\xc7\x9f\xe2\xd9\xed\xcd\x7c\xa2\x24\x0a\xd5\x5f\xc8\xcf\xfb\xe2\x42\x85\x47\x9d\xfc\x67\x9d\x7e\xdb\x8d\x8e\xf2\xff\x07\xc0\x74\x11\xdf\xcd\x57\xd1\x87\x9b\xf9\x6c\x32\x42\xdf\xff\xc9\xf9\x43\xff\x7c\xc4\x22\x8e\x93\xc9\xf9\x43\x9c\x3c\xf6\x27\xd3\xeb\xc9\xf9\x83\x7b\x3c\x3e\xd5\xfd\xbf\x45\xb7\x42\xe6\xcf\x57\xf1\x25\xaf\x9f\x1a\xfa\x3d\xe7\x67\xf8\xb4\xa7\x0c\x59\x83\x87\x47\x3c\xeb\xfa\x02\x1b\xdf\xde\x0f\x1c\xa4\xd7\x0f\x0c\xd7\xf5\xd0\x91\x11\xd2\x58\x5e\x55\x6e\x7f\x99\xa7\x42\x61\xef\xfa\xbb\x3e\xc4\x28\xfc\xd5\x1a\x8b\x9e\x88\xf3\x53\xbf\xdf\xb8\xcc\x8f\x9b\x4a\x58\xd7\x79\x5b\xd2\x11\x59\x2b\x65\x5d\x4a\xe7\xcc\x23\x9f\x43\x9c\xd1\xb6\x5a\xd5\xc7\x0d\x69\x4b\xad\x3a\xf0\x8e\x1f\x90\x29\xe9\xf6\x1e\xb9\xf9\x76\x03\xdb\x07\x71\x18\xcb\x65\xce\x75\x8e\x4a\x65\xfd\x88\x06\x47\xf8\x2f\x42\xe6\xaa\x33\x70\xf7\x6d\xad\xdb\x61\x06\x25\xdf\x11\x02\xda\xbb\xa5\x6b\x49\x3a\xeb\x0f\xd1\x95\x22\x2b\xdd\x66\xea\xca\x03\x3a\xfa\x49\x13\x0c\x71\x9d\x95\xce\xe9\x2f\x04\x64\x2c\xf8\xe3\x3a\x5e\x7d\xbd\x42\x2a\x24\xfb\x27\x00\x00\xff\xff\xc5\xbd\xc1\x5c\x45\x06\x00\x00") +var _testImagesSampleApiserverMakefile = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x54\x61\x6f\xdb\x36\x10\xfd\x1c\xfe\x8a\x07\x2b\xe8\x9a\xc1\x92\xec\x60\x6b\x87\x14\xc6\xa6\xda\x9e\x63\x34\xb5\x06\xcb\x69\xd1\xa1\x40\x40\x49\x67\x89\x8b\x44\x6a\x24\x65\xd9\x08\xf2\xdf\x07\xca\x4e\xba\x22\x18\x30\x7f\x91\x25\xde\xbd\x7b\x7c\xf7\xee\x3c\x4c\x55\x73\xd0\xa2\x28\x2d\x2e\x47\x97\x23\x6c\x4a\xc2\x87\x36\x25\x2d\xc9\x92\x41\xd4\xda\x52\x69\x13\x30\x8f\x79\xb8\x11\x19\x49\x43\x39\x5a\x99\x93\x86\x2d\x09\x51\xc3\xb3\x92\x9e\x4e\x86\xf8\x44\xda\x08\x25\x71\x19\x8c\xf0\xda\x05\x0c\x4e\x47\x83\x8b\x77\xcc\xc3\x41\xb5\xa8\xf9\x01\x52\x59\xb4\x86\x60\x4b\x61\xb0\x15\x15\x81\xf6\x19\x35\x16\x42\x22\x53\x75\x53\x09\x2e\x33\x42\x27\x6c\xd9\x97\x39\x81\x04\xcc\xc3\x97\x13\x84\x4a\x2d\x17\x12\x1c\x99\x6a\x0e\x50\xdb\x7f\xc7\x81\xdb\x9e\xb0\xfb\x95\xd6\x36\x57\x61\xd8\x75\x5d\xc0\x7b\xb2\x81\xd2\x45\x58\x1d\x03\x4d\x78\xb3\x9c\xce\x57\xc9\xdc\xbf\x0c\x46\x7d\xca\xad\xac\xc8\x18\x68\xfa\xbb\x15\x9a\x72\xa4\x07\xf0\xa6\xa9\x44\xc6\xd3\x8a\x50\xf1\x0e\x4a\x83\x17\x9a\x28\x87\x55\x8e\x6f\xa7\x85\x15\xb2\x18\xc2\xa8\xad\xed\xb8\x26\xe6\x21\x17\xc6\x6a\x91\xb6\xf6\x3b\xb1\x9e\xd8\x09\xf3\x5d\x80\x92\xe0\x12\x83\x28\xc1\x32\x19\xe0\x7d\x94\x2c\x93\x21\xf3\xf0\x79\xb9\xb9\x8e\x6f\x37\xf8\x1c\xad\xd7\xd1\x6a\xb3\x9c\x27\x88\xd7\x98\xc6\xab\xd9\x72\xb3\x8c\x57\x09\xe2\xdf\x11\xad\xbe\xe0\xc3\x72\x35\x1b\x82\x84\x2d\x49\x83\xf6\x8d\x76\xfc\x95\x86\x70\x32\x52\xee\x34\x4b\x88\xbe\x23\xb0\x55\x47\x42\xa6\xa1\x4c\x6c\x45\x86\x8a\xcb\xa2\xe5\x05\xa1\x50\x3b\xd2\x52\xc8\x02\x0d\xe9\x5a\x18\xd7\x4c\x03\x2e\x73\xe6\xa1\x12\xb5\xb0\xdc\xf6\x5f\x5e\x5c\x2a\x60\x2c\x4e\xf0\xeb\x04\x95\x90\xed\x9e\x45\xeb\xe9\xb5\x7b\xe3\x75\xfe\xe6\x27\xb6\x89\xd6\x8b\xf9\xc6\xbd\x9f\xbf\x9e\xde\xae\x67\xcb\xf5\x05\x5b\xc4\xd1\xfa\x23\x26\x78\xcb\x16\xf1\x4d\xb4\x5a\xdc\x7d\x9a\xaf\x93\x65\xbc\xea\x31\xb8\x25\x63\x59\xb2\x9e\xde\xcd\x96\x6b\xb8\x34\xa9\x6c\x2e\x34\xce\x5f\x9b\x92\xaa\x0a\x4d\x97\x5f\x5c\x30\xda\x37\x4a\x5b\xc6\x3c\xbc\x6f\x45\x95\x63\x37\x0e\xc6\x6f\x83\x91\x6b\x0c\x49\xd3\xea\xe3\xad\xb3\x56\x6b\x92\x16\x9a\x2a\xe2\x86\x60\xda\xc6\xa5\x19\x70\x34\x5a\x28\x8d\xdd\xc9\xb4\x27\x13\x19\x5e\x37\x15\x81\x37\xc2\x90\xde\x91\x66\x1e\x16\x64\x7b\x33\xaa\xd6\x22\x75\xa5\x9c\x42\x56\xa1\x51\x4d\xeb\xc8\xa2\x56\x79\x5b\x11\x32\x67\x30\xe6\xb9\x29\x92\x43\x14\xa7\x2c\xc4\x49\xe8\x04\xf1\x9f\xf5\x26\xb9\x73\xe9\x3d\x14\x4b\x85\xbc\x62\x67\xb9\xca\xee\x49\x43\xb7\x12\xbe\xaf\x6b\xf8\x02\xfe\x0e\x83\xf3\x87\xa3\x78\x8f\x57\xdf\xfe\xfd\x39\xc0\xfd\x2f\x26\x28\x32\x1d\x08\x15\xf6\x20\xbe\xa8\x79\x41\xe1\x7d\x9b\x92\x9f\x69\x65\xcc\x95\xd3\xe2\x4d\xf0\xb3\x3f\xc6\x57\x76\x76\x16\xa6\x42\x86\x29\x37\x25\xfc\x0c\x03\xf7\xe5\xac\xbe\x77\x82\xfa\x0d\xc2\x42\x85\x46\x67\xfd\x33\x15\x12\xaf\x5e\xf5\x29\x67\x8b\x78\x3c\x1e\x7f\x8c\x67\xb7\x37\xf3\x89\x92\x28\x54\x7f\x21\x3f\xef\x8b\x0b\x15\x1e\x75\xf2\x9f\x75\xfa\x6d\x37\x3a\xca\xff\x1f\x00\xd3\x45\x7c\x37\x5f\x45\xef\x6f\xe6\xb3\xc9\x08\x7d\xff\x27\xe7\x0f\xfd\xf3\x11\x8b\x38\x4e\x26\xe7\x0f\x71\xf2\xd8\x9f\x4c\xaf\x27\xe7\x0f\xee\xf1\xf8\x54\xf7\xff\x16\xdd\x0a\x99\x3f\x5f\xc5\x97\xbc\x7e\x6a\xe8\xb7\x9c\x1f\xe1\xd3\x9e\x32\x64\x0d\x1e\x1e\xf1\xac\xeb\x0b\x6c\x7c\x7d\x37\x70\x90\x5e\x3f\x30\x5c\xd7\x43\x47\x46\x48\x63\x79\x55\xb9\xfd\x65\x9e\x0a\x85\xbd\xeb\xef\xfa\x10\xa3\xf0\x57\x6b\x2c\x7a\x22\xce\x4f\xfd\x7e\xe3\x32\x3f\x6e\x2a\x61\x5d\xe7\x6d\x49\x47\x64\xad\x94\x75\x29\x9d\x33\x8f\x7c\x0e\x71\x46\xdb\x6a\x55\x1f\x37\xa4\x2d\xb5\xea\xc0\x3b\x7e\x40\xa6\xa4\xdb\x7b\xe4\xe6\xdb\x0d\x6c\x1f\xc4\x61\x2c\x97\x39\xd7\x39\x2a\x95\xf5\x23\x1a\x1c\xe1\x3f\x0b\x99\xab\xce\xc0\xdd\xb7\xb5\x6e\x87\x19\x94\x7c\x47\x08\x68\xef\x96\xae\x25\xe9\xac\x3f\x44\x57\x8a\xac\x74\x9b\xa9\x2b\x0f\xe8\xe8\x07\x4d\x30\xc4\x75\x56\x3a\xa7\xbf\x10\x90\xb1\xe0\x8f\xeb\x78\xf5\xe5\x0a\xa9\x90\xec\x9f\x00\x00\x00\xff\xff\x86\xa2\x77\x62\x45\x06\x00\x00") func testImagesSampleApiserverMakefileBytes() ([]byte, error) { return bindataRead( diff --git a/test/e2e/scheduling/priorities.go b/test/e2e/scheduling/priorities.go index 9a5696aa8fad7..4cf9eafbccc7c 100644 --- a/test/e2e/scheduling/priorities.go +++ b/test/e2e/scheduling/priorities.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "math" + "sync" "time" "github.com/onsi/ginkgo" @@ -33,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" @@ -527,6 +529,9 @@ func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, n } } + errChan := make(chan error, len(nodes)) + var wg sync.WaitGroup + // we need the max one to keep the same cpu/mem use rate ratio = math.Max(maxCPUFraction, maxMemFraction) for _, node := range nodes { @@ -566,14 +571,27 @@ func createBalancedPodForNodes(f *framework.Framework, cs clientset.Interface, n }, }, } - - err := testutils.StartPods(cs, 1, ns, string(uuid.NewUUID()), - *initPausePod(f, *podConfig), true, framework.Logf) - + wg.Add(1) + go func() { + defer wg.Done() + err := testutils.StartPods(cs, 1, ns, string(uuid.NewUUID()), + *initPausePod(f, *podConfig), true, framework.Logf) + if err != nil { + errChan <- err + } + }() + } + wg.Wait() + close(errChan) + var errs []error + for err := range errChan { if err != nil { - return cleanUp, err + errs = append(errs, err) } } + if len(errs) > 0 { + return cleanUp, errors.NewAggregate(errs) + } nodeNameToPodList = podListForEachNode(cs) for _, node := range nodes { diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go index 9b01c741879c1..ac0eacf3f84dd 100644 --- a/test/e2e/storage/volume_provisioning.go +++ b/test/e2e/storage/volume_provisioning.go @@ -825,6 +825,7 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() { ginkgo.It("should report an error and create no PV", func() { e2eskipper.SkipUnlessProviderIs("aws") test := testsuites.StorageClassTest{ + Client: c, Name: "AWS EBS with invalid KMS key", Provisioner: "kubernetes.io/aws-ebs", Timeouts: f.Timeouts, diff --git a/test/e2e/testing-manifests/scheduling/nvidia-driver-installer.yaml b/test/e2e/testing-manifests/scheduling/nvidia-driver-installer.yaml index e74440c2aae48..f4a6b908e3fc4 100644 --- a/test/e2e/testing-manifests/scheduling/nvidia-driver-installer.yaml +++ b/test/e2e/testing-manifests/scheduling/nvidia-driver-installer.yaml @@ -48,7 +48,12 @@ spec: hostPath: path: / initContainers: - - image: gcr.io/cos-cloud/cos-gpu-installer:v20200701 + # The COS GPU installer image version may be dependent on the version of COS being used. + # Refer to details about the installer in https://cos.googlesource.com/cos/tools/+/refs/heads/master/src/cmd/cos_gpu_installer/ + # and the COS release notes (https://cloud.google.com/container-optimized-os/docs/release-notes) to determine version COS GPU installer for a given version of COS. + + # Maps to gcr.io/cos-cloud/cos-gpu-installer:v2.0.3 - suitable for COS M85 as per https://cloud.google.com/container-optimized-os/docs/release-notes#cos-85-13310-1209-3 + - image: gcr.io/cos-cloud/cos-gpu-installer@sha256:1cf2701dc2c3944a93fd06cb6c9eedfabf323425483ba3af294510621bb37d0e name: nvidia-driver-installer resources: requests: diff --git a/test/e2e_kubeadm/kubeadm_config_test.go b/test/e2e_kubeadm/kubeadm_config_test.go index 693e62b76c103..3e4b3677072fa 100644 --- a/test/e2e_kubeadm/kubeadm_config_test.go +++ b/test/e2e_kubeadm/kubeadm_config_test.go @@ -32,7 +32,6 @@ const ( kubeadmConfigRoleName = "kubeadm:nodes-kubeadm-config" kubeadmConfigRoleBindingName = kubeadmConfigRoleName kubeadmConfigClusterConfigurationConfigMapKey = "ClusterConfiguration" - kubeadmConfigClusterStatusConfigMapKey = "ClusterStatus" ) var ( @@ -60,23 +59,6 @@ var _ = Describe("kubeadm-config ConfigMap", func() { cm := GetConfigMap(f.ClientSet, kubeSystemNamespace, kubeadmConfigName) gomega.Expect(cm.Data).To(gomega.HaveKey(kubeadmConfigClusterConfigurationConfigMapKey)) - gomega.Expect(cm.Data).To(gomega.HaveKey(kubeadmConfigClusterStatusConfigMapKey)) - - m := unmarshalYaml(cm.Data[kubeadmConfigClusterStatusConfigMapKey]) - if _, ok := m["apiEndpoints"]; ok { - d := m["apiEndpoints"].(map[interface{}]interface{}) - // get all control-plane nodes - controlPlanes := getControlPlaneNodes(f.ClientSet) - - // checks that all the control-plane nodes are in the apiEndpoints list - for _, cp := range controlPlanes.Items { - if _, ok := d[cp.Name]; !ok { - framework.Failf("failed to get apiEndpoints for control-plane %s in %s", cp.Name, kubeadmConfigClusterStatusConfigMapKey) - } - } - } else { - framework.Failf("failed to get apiEndpoints from %s", kubeadmConfigClusterStatusConfigMapKey) - } }) ginkgo.It("should have related Role and RoleBinding", func() { diff --git a/test/images/Makefile b/test/images/Makefile index b73ae0eb689e5..54862dba9d1e1 100644 --- a/test/images/Makefile +++ b/test/images/Makefile @@ -16,7 +16,7 @@ REGISTRY ?= gcr.io/kubernetes-e2e-test-images GOARM ?= 7 DOCKER_CERT_BASE_PATH ?= QEMUVERSION=v5.1.0-2 -GOLANG_VERSION=1.16.4 +GOLANG_VERSION=1.16.5 export ifndef WHAT diff --git a/test/images/sample-apiserver/Makefile b/test/images/sample-apiserver/Makefile index 67b76b08c59f0..984b9501377eb 100644 --- a/test/images/sample-apiserver/Makefile +++ b/test/images/sample-apiserver/Makefile @@ -24,7 +24,7 @@ export # Get without building to populate module cache # Then, get with OS/ARCH-specific env to build bin: - docker run --rm -i -v "${TARGET}:${TARGET}:Z" k8s.gcr.io/build-image/kube-cross:v1.16.4-1 \ + docker run --rm -i -v "${TARGET}:${TARGET}:Z" k8s.gcr.io/build-image/kube-cross:v1.16.5-1 \ /bin/bash -c "\ mkdir -p /go/src /go/bin && \ GO111MODULE=on go get -d k8s.io/sample-apiserver@v0.17.0 && \ diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 66348582dd536..b398671fca2ac 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -223,7 +223,7 @@ func initImageConfigs() (map[int]Config, map[int]Config) { configs[CheckMetadataConcealment] = Config{promoterE2eRegistry, "metadata-concealment", "1.6"} configs[CudaVectorAdd] = Config{e2eRegistry, "cuda-vector-add", "1.0"} configs[CudaVectorAdd2] = Config{promoterE2eRegistry, "cuda-vector-add", "2.2"} - configs[DebianIptables] = Config{buildImageRegistry, "debian-iptables", "buster-v1.6.0"} + configs[DebianIptables] = Config{buildImageRegistry, "debian-iptables", "buster-v1.6.2"} configs[EchoServer] = Config{promoterE2eRegistry, "echoserver", "2.3"} configs[Etcd] = Config{gcEtcdRegistry, "etcd", "3.4.13-0"} configs[GlusterDynamicProvisioner] = Config{promoterE2eRegistry, "glusterdynamic-provisioner", "v1.0"} diff --git a/vendor/modules.txt b/vendor/modules.txt index f015e8b0e5d9c..daf50a791fcc3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -200,7 +200,6 @@ github.com/chai2010/gettext-go/gettext github.com/chai2010/gettext-go/gettext/mo github.com/chai2010/gettext-go/gettext/plural github.com/chai2010/gettext-go/gettext/po -# github.com/checkpoint-restore/go-criu/v4 => github.com/checkpoint-restore/go-criu/v4 v4.1.0 # github.com/checkpoint-restore/go-criu/v5 v5.0.0 => github.com/checkpoint-restore/go-criu/v5 v5.0.0 github.com/checkpoint-restore/go-criu/v5 # github.com/checkpoint-restore/go-criu/v5 => github.com/checkpoint-restore/go-criu/v5 v5.0.0 @@ -923,7 +922,7 @@ github.com/opencontainers/runtime-spec/specs-go github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalk -# github.com/openshift/api v0.0.0-20210521075222-e273a339932a => github.com/openshift/api v0.0.0-20210422150128-d8a48168c81c +# github.com/openshift/api v0.0.0-20210610130314-a6ac319a7eed => github.com/openshift/api v0.0.0-20210422150128-d8a48168c81c ## explicit # github.com/openshift/api => github.com/openshift/api v0.0.0-20210422150128-d8a48168c81c github.com/openshift/api/apiserver/v1 @@ -1078,7 +1077,7 @@ github.com/openshift/client-go/user/informers/externalversions/internalinterface github.com/openshift/client-go/user/informers/externalversions/user github.com/openshift/client-go/user/informers/externalversions/user/v1 github.com/openshift/client-go/user/listers/user/v1 -# github.com/openshift/library-go v0.0.0-20210521084623-7392ea9b02ca => github.com/openshift/library-go v0.0.0-20210407092538-7021fda6f427 +# github.com/openshift/library-go v0.0.0-20210611143017-0d0ef669a361 => github.com/openshift/library-go v0.0.0-20210407092538-7021fda6f427 ## explicit # github.com/openshift/library-go => github.com/openshift/library-go v0.0.0-20210407092538-7021fda6f427 github.com/openshift/library-go/pkg/apiserver/admission/admissionregistrationtesting @@ -2814,8 +2813,9 @@ k8s.io/utils/trace # modernc.org/strutil => modernc.org/strutil v1.0.0 # modernc.org/xc => modernc.org/xc v1.0.0 # rsc.io/pdf => rsc.io/pdf v0.1.1 -# sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 -# sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 +# sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 +## explicit +# sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client # sigs.k8s.io/controller-tools => sigs.k8s.io/controller-tools v0.2.8 diff --git a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go index 1d9a4950024ec..9d1f6f911e0ae 100644 --- a/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go +++ b/vendor/sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client/client.go @@ -19,6 +19,7 @@ package client import ( "context" "errors" + "fmt" "io" "math/rand" "net" @@ -49,6 +50,10 @@ type grpcTunnel struct { conns map[int64]*conn pendingDialLock sync.RWMutex connsLock sync.RWMutex + + // The tunnel will be closed if the caller fails to read via conn.Read() + // more than readTimeoutSeconds after a packet has been received. + readTimeoutSeconds int } type clientConn interface { @@ -75,9 +80,10 @@ func CreateSingleUseGrpcTunnel(address string, opts ...grpc.DialOption) (Tunnel, } tunnel := &grpcTunnel{ - stream: stream, - pendingDial: make(map[int64]chan<- dialResult), - conns: make(map[int64]*conn), + stream: stream, + pendingDial: make(map[int64]chan<- dialResult), + conns: make(map[int64]*conn), + readTimeoutSeconds: 10, } go tunnel.serve(c) @@ -110,10 +116,17 @@ func (t *grpcTunnel) serve(c clientConn) { if !ok { klog.V(1).Infoln("DialResp not recognized; dropped") } else { - ch <- dialResult{ + result := dialResult{ err: resp.Error, connid: resp.ConnectID, } + select { + case ch <- result: + default: + klog.ErrorS(fmt.Errorf("blocked pending channel"), "Received second dial response for connection request", "connectionID", resp.ConnectID, "dialID", resp.Random) + // On multiple dial responses, avoid leaking serve goroutine. + return + } } if resp.Error != "" { @@ -129,7 +142,14 @@ func (t *grpcTunnel) serve(c clientConn) { t.connsLock.RUnlock() if ok { - conn.readCh <- resp.Data + timer := time.NewTimer((time.Duration)(t.readTimeoutSeconds) * time.Second) + select { + case conn.readCh <- resp.Data: + timer.Stop() + case <-timer.C: + klog.ErrorS(fmt.Errorf("timeout"), "readTimeout has been reached, the grpc connection to the proxy server will be closed", "connectionID", conn.connID, "readTimeoutSeconds", t.readTimeoutSeconds) + return + } } else { klog.V(1).InfoS("connection not recognized", "connectionID", resp.ConnectID) } @@ -160,8 +180,8 @@ func (t *grpcTunnel) Dial(protocol, address string) (net.Conn, error) { return nil, errors.New("protocol not supported") } - random := rand.Int63() - resCh := make(chan dialResult) + random := rand.Int63() /* #nosec G404 */ + resCh := make(chan dialResult, 1) t.pendingDialLock.Lock() t.pendingDial[random] = resCh t.pendingDialLock.Unlock() @@ -199,7 +219,7 @@ func (t *grpcTunnel) Dial(protocol, address string) (net.Conn, error) { } c.connID = res.connid c.readCh = make(chan []byte, 10) - c.closeCh = make(chan string) + c.closeCh = make(chan string, 1) t.connsLock.Lock() t.conns[res.connid] = c t.connsLock.Unlock()