From a4ac2e3b63b8793f465f35cca2ffff4bba9d5ad8 Mon Sep 17 00:00:00 2001 From: Eric Lin Date: Thu, 5 Jun 2025 21:01:17 +0000 Subject: [PATCH 1/6] Clean backoff record earlier Once received job deletion event, it cleans the backoff records for that job before enqueueing this job so that we can avoid a race condition that the syncJob() may incorrect use stale backoff records for a newly created job with same key. Co-authored-by: Michal Wozniak --- pkg/controller/job/job_controller.go | 5 ++ test/integration/job/job_test.go | 97 ++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) diff --git a/pkg/controller/job/job_controller.go b/pkg/controller/job/job_controller.go index 28d713a3a8754..fb2716590e29e 100644 --- a/pkg/controller/job/job_controller.go +++ b/pkg/controller/job/job_controller.go @@ -521,6 +521,11 @@ func (jm *Controller) deleteJob(logger klog.Logger, obj interface{}) { } } jm.cleanupPodFinalizers(jobObj) + key := cache.MetaObjectToName(jobObj).String() + err := jm.podBackoffStore.removeBackoffRecord(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("error removing backoff record %w", err)) + } } // enqueueSyncJobImmediately tells the Job controller to invoke syncJob diff --git a/test/integration/job/job_test.go b/test/integration/job/job_test.go index 64d08d16ca7fc..cd76b3ffa0f96 100644 --- a/test/integration/job/job_test.go +++ b/test/integration/job/job_test.go @@ -2198,6 +2198,103 @@ func TestManagedBy_Reenabling(t *testing.T) { }) } +// TestImmediateJobRecreation verifies that the replacement Job creates the Pods +// quickly after re-creation, see https://github.com/kubernetes/kubernetes/issues/132042. +func TestImmediateJobRecreation(t *testing.T) { + // set the backoff delay very high to make sure the test does not pass waiting long on asserts + t.Cleanup(setDurationDuringTest(&jobcontroller.DefaultJobPodFailureBackOff, 2*wait.ForeverTestTimeout)) + closeFn, restConfig, clientSet, ns := setup(t, "recreate-job-immediately") + t.Cleanup(closeFn) + ctx, cancel := startJobControllerAndWaitForCaches(t, restConfig) + t.Cleanup(cancel) + + baseJob := batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + }, + Spec: batchv1.JobSpec{ + Completions: ptr.To[int32](1), + Parallelism: ptr.To[int32](1), + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "main-container", + Image: "foo", + }, + }, + }, + }, + }, + } + jobSpec := func(idx int) batchv1.Job { + spec := baseJob.DeepCopy() + spec.Name = fmt.Sprintf("test-job-%d", idx) + return *spec + } + + var jobObjs []*batchv1.Job + // We create multiple Jobs to make the repro more likely. In particular, we need + // more Jobs than the number of Job controller workers to make it very unlikely + // that syncJob executes (and cleans the in-memory state) before the corresponding + // replacement Jobs are created. + for i := 0; i < 3; i++ { + jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, ptr.To(jobSpec(i))) + if err != nil { + t.Fatalf("Error %v when creating the job %q", err, klog.KObj(jobObj)) + } + jobObjs = append(jobObjs, jobObj) + } + + for _, jobObj := range jobObjs { + validateJobsPodsStatusOnly(ctx, t, clientSet, jobObj, podsByStatus{ + Active: 1, + Ready: ptr.To[int32](0), + Terminating: ptr.To[int32](0), + }) + + if _, err := setJobPodsPhase(ctx, clientSet, jobObj, v1.PodFailed, 1); err != nil { + t.Fatalf("Error %v when setting phase %s on the pod of job %v", err, v1.PodFailed, klog.KObj(jobObj)) + } + + // Await to account for the failed Pod + validateJobsPodsStatusOnly(ctx, t, clientSet, jobObj, podsByStatus{ + Failed: 1, + Ready: ptr.To[int32](0), + Terminating: ptr.To[int32](0), + }) + } + + for i := 0; i < len(jobObjs); i++ { + jobObj := jobObjs[i] + jobClient := clientSet.BatchV1().Jobs(jobObj.Namespace) + if err := jobClient.Delete(ctx, jobObj.Name, metav1.DeleteOptions{ + // Use propagationPolicy=background so that we don't need to wait for the job object to be gone. + PropagationPolicy: ptr.To(metav1.DeletePropagationBackground), + }); err != nil { + t.Fatalf("Error %v when deleting the job %v", err, klog.KObj(jobObj)) + } + + // re-create the job immediately + jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, ptr.To(jobSpec(i))) + if err != nil { + t.Fatalf("Error %q while creating the job %q", err, klog.KObj(jobObj)) + } + jobObjs[i] = jobObj + } + + // total timeout (3*5s) is less than 2*ForeverTestTimeout. + for _, jobObj := range jobObjs { + // wait maks 5s for the Active=1. This assert verifies that the backoff + // delay is not applied to the replacement instance of the Job. + validateJobsPodsStatusOnlyWithTimeout(ctx, t, clientSet, jobObj, podsByStatus{ + Active: 1, + Ready: ptr.To[int32](0), + Terminating: ptr.To[int32](0), + }, 5*time.Second) + } +} + // TestManagedBy_RecreatedJob verifies that the Job controller skips // reconciliation of a job with managedBy field, when this is a recreated job, // and there is still a pending sync queued for the previous job. From 152efa92686e41cbadfcf7aacb4094193a153a09 Mon Sep 17 00:00:00 2001 From: bconry Date: Tue, 18 Mar 2025 20:18:12 +0000 Subject: [PATCH 2/6] Add check to see if promote worked within the retry loop --- cmd/kubeadm/app/util/etcd/etcd.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmd/kubeadm/app/util/etcd/etcd.go b/cmd/kubeadm/app/util/etcd/etcd.go index 480d779df251f..8dba40b85d173 100644 --- a/cmd/kubeadm/app/util/etcd/etcd.go +++ b/cmd/kubeadm/app/util/etcd/etcd.go @@ -576,6 +576,15 @@ func (c *Client) MemberPromote(learnerID uint64) error { ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) defer cancel() + isLearner, err := c.isLearner(learnerID) + if err != nil { + return false, err + } + if !isLearner { + klog.V(1).Infof("[etcd] Member %s was already promoted.", strconv.FormatUint(learnerID, 16)) + return true, nil + } + _, err = cli.MemberPromote(ctx, learnerID) if err == nil { klog.V(1).Infof("[etcd] The learner was promoted as a voting member: %s", strconv.FormatUint(learnerID, 16)) From ef4a037da2f123df1d40ada4a6bb46e6254cbfb0 Mon Sep 17 00:00:00 2001 From: Kubernetes Release Robot Date: Tue, 17 Jun 2025 18:55:04 +0000 Subject: [PATCH 3/6] Update CHANGELOG/CHANGELOG-1.31.md for v1.31.10 --- CHANGELOG/CHANGELOG-1.31.md | 276 +++++++++++++++++++++++++----------- 1 file changed, 193 insertions(+), 83 deletions(-) diff --git a/CHANGELOG/CHANGELOG-1.31.md b/CHANGELOG/CHANGELOG-1.31.md index 1bd0c51238c34..0de66a3ae76bf 100644 --- a/CHANGELOG/CHANGELOG-1.31.md +++ b/CHANGELOG/CHANGELOG-1.31.md @@ -1,234 +1,233 @@ -- [v1.31.9](#v1319) - - [Downloads for v1.31.9](#downloads-for-v1319) +- [v1.31.10](#v13110) + - [Downloads for v1.31.10](#downloads-for-v13110) - [Source Code](#source-code) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - [Container Images](#container-images) - - [Changelog since v1.31.8](#changelog-since-v1318) + - [Changelog since v1.31.9](#changelog-since-v1319) - [Changes by Kind](#changes-by-kind) - [Feature](#feature) - [Bug or Regression](#bug-or-regression) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake) - [Dependencies](#dependencies) - [Added](#added) - [Changed](#changed) - [Removed](#removed) -- [v1.31.8](#v1318) - - [Downloads for v1.31.8](#downloads-for-v1318) +- [v1.31.9](#v1319) + - [Downloads for v1.31.9](#downloads-for-v1319) - [Source Code](#source-code-1) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - [Container Images](#container-images-1) - - [Changelog since v1.31.7](#changelog-since-v1317) + - [Changelog since v1.31.8](#changelog-since-v1318) - [Changes by Kind](#changes-by-kind-1) + - [Feature](#feature-1) - [Bug or Regression](#bug-or-regression-1) - [Dependencies](#dependencies-1) - [Added](#added-1) - [Changed](#changed-1) - [Removed](#removed-1) -- [v1.31.7](#v1317) - - [Downloads for v1.31.7](#downloads-for-v1317) +- [v1.31.8](#v1318) + - [Downloads for v1.31.8](#downloads-for-v1318) - [Source Code](#source-code-2) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) - [Container Images](#container-images-2) - - [Changelog since v1.31.6](#changelog-since-v1316) + - [Changelog since v1.31.7](#changelog-since-v1317) - [Changes by Kind](#changes-by-kind-2) - [Bug or Regression](#bug-or-regression-2) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake) - [Dependencies](#dependencies-2) - [Added](#added-2) - [Changed](#changed-2) - [Removed](#removed-2) -- [v1.31.6](#v1316) - - [Downloads for v1.31.6](#downloads-for-v1316) +- [v1.31.7](#v1317) + - [Downloads for v1.31.7](#downloads-for-v1317) - [Source Code](#source-code-3) - [Client Binaries](#client-binaries-3) - [Server Binaries](#server-binaries-3) - [Node Binaries](#node-binaries-3) - [Container Images](#container-images-3) - - [Changelog since v1.31.5](#changelog-since-v1315) - - [Important Security Information](#important-security-information) - - [CVE-2025-0426: Node Denial of Service via Kubelet Checkpoint API](#cve-2025-0426-node-denial-of-service-via-kubelet-checkpoint-api) + - [Changelog since v1.31.6](#changelog-since-v1316) - [Changes by Kind](#changes-by-kind-3) - - [Feature](#feature-1) - [Bug or Regression](#bug-or-regression-3) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-1) - [Dependencies](#dependencies-3) - [Added](#added-3) - [Changed](#changed-3) - [Removed](#removed-3) -- [v1.31.5](#v1315) - - [Downloads for v1.31.5](#downloads-for-v1315) +- [v1.31.6](#v1316) + - [Downloads for v1.31.6](#downloads-for-v1316) - [Source Code](#source-code-4) - [Client Binaries](#client-binaries-4) - [Server Binaries](#server-binaries-4) - [Node Binaries](#node-binaries-4) - [Container Images](#container-images-4) - - [Changelog since v1.31.4](#changelog-since-v1314) - - [Important Security Information](#important-security-information-1) - - [CVE-2024-9042: Command Injection affecting Windows nodes via nodes/*/logs/query API](#cve-2024-9042-command-injection-affecting-windows-nodes-via-nodeslogsquery-api) + - [Changelog since v1.31.5](#changelog-since-v1315) + - [Important Security Information](#important-security-information) + - [CVE-2025-0426: Node Denial of Service via Kubelet Checkpoint API](#cve-2025-0426-node-denial-of-service-via-kubelet-checkpoint-api) - [Changes by Kind](#changes-by-kind-4) - - [API Change](#api-change) - [Feature](#feature-2) - [Bug or Regression](#bug-or-regression-4) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-2) - [Dependencies](#dependencies-4) - [Added](#added-4) - [Changed](#changed-4) - [Removed](#removed-4) -- [v1.31.4](#v1314) - - [Downloads for v1.31.4](#downloads-for-v1314) +- [v1.31.5](#v1315) + - [Downloads for v1.31.5](#downloads-for-v1315) - [Source Code](#source-code-5) - [Client Binaries](#client-binaries-5) - [Server Binaries](#server-binaries-5) - [Node Binaries](#node-binaries-5) - [Container Images](#container-images-5) - - [Changelog since v1.31.3](#changelog-since-v1313) + - [Changelog since v1.31.4](#changelog-since-v1314) + - [Important Security Information](#important-security-information-1) + - [CVE-2024-9042: Command Injection affecting Windows nodes via nodes/*/logs/query API](#cve-2024-9042-command-injection-affecting-windows-nodes-via-nodeslogsquery-api) - [Changes by Kind](#changes-by-kind-5) + - [API Change](#api-change) - [Feature](#feature-3) - [Bug or Regression](#bug-or-regression-5) - [Dependencies](#dependencies-5) - [Added](#added-5) - [Changed](#changed-5) - [Removed](#removed-5) -- [v1.31.3](#v1313) - - [Downloads for v1.31.3](#downloads-for-v1313) +- [v1.31.4](#v1314) + - [Downloads for v1.31.4](#downloads-for-v1314) - [Source Code](#source-code-6) - [Client Binaries](#client-binaries-6) - [Server Binaries](#server-binaries-6) - [Node Binaries](#node-binaries-6) - [Container Images](#container-images-6) - - [Changelog since v1.31.2](#changelog-since-v1312) + - [Changelog since v1.31.3](#changelog-since-v1313) - [Changes by Kind](#changes-by-kind-6) + - [Feature](#feature-4) - [Bug or Regression](#bug-or-regression-6) - [Dependencies](#dependencies-6) - [Added](#added-6) - [Changed](#changed-6) - [Removed](#removed-6) -- [v1.31.2](#v1312) - - [Downloads for v1.31.2](#downloads-for-v1312) +- [v1.31.3](#v1313) + - [Downloads for v1.31.3](#downloads-for-v1313) - [Source Code](#source-code-7) - [Client Binaries](#client-binaries-7) - [Server Binaries](#server-binaries-7) - [Node Binaries](#node-binaries-7) - [Container Images](#container-images-7) - - [Changelog since v1.31.1](#changelog-since-v1311) + - [Changelog since v1.31.2](#changelog-since-v1312) - [Changes by Kind](#changes-by-kind-7) - - [Feature](#feature-4) - [Bug or Regression](#bug-or-regression-7) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-2) - - [Uncategorized](#uncategorized) - [Dependencies](#dependencies-7) - [Added](#added-7) - [Changed](#changed-7) - [Removed](#removed-7) -- [v1.31.1](#v1311) - - [Downloads for v1.31.1](#downloads-for-v1311) +- [v1.31.2](#v1312) + - [Downloads for v1.31.2](#downloads-for-v1312) - [Source Code](#source-code-8) - [Client Binaries](#client-binaries-8) - [Server Binaries](#server-binaries-8) - [Node Binaries](#node-binaries-8) - [Container Images](#container-images-8) - - [Changelog since v1.31.0](#changelog-since-v1310) + - [Changelog since v1.31.1](#changelog-since-v1311) - [Changes by Kind](#changes-by-kind-8) - - [Deprecation](#deprecation) - - [API Change](#api-change-1) - [Feature](#feature-5) - [Bug or Regression](#bug-or-regression-8) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-3) + - [Uncategorized](#uncategorized) - [Dependencies](#dependencies-8) - [Added](#added-8) - [Changed](#changed-8) - [Removed](#removed-8) -- [v1.31.0](#v1310) - - [Downloads for v1.31.0](#downloads-for-v1310) +- [v1.31.1](#v1311) + - [Downloads for v1.31.1](#downloads-for-v1311) - [Source Code](#source-code-9) - [Client Binaries](#client-binaries-9) - [Server Binaries](#server-binaries-9) - [Node Binaries](#node-binaries-9) - [Container Images](#container-images-9) - - [Changelog since v1.30.0](#changelog-since-v1300) - - [Urgent Upgrade Notes](#urgent-upgrade-notes) - - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade) + - [Changelog since v1.31.0](#changelog-since-v1310) - [Changes by Kind](#changes-by-kind-9) - - [Deprecation](#deprecation-1) - - [API Change](#api-change-2) + - [Deprecation](#deprecation) + - [API Change](#api-change-1) - [Feature](#feature-6) - - [Failing Test](#failing-test) - [Bug or Regression](#bug-or-regression-9) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-4) - [Dependencies](#dependencies-9) - [Added](#added-9) - [Changed](#changed-9) - [Removed](#removed-9) -- [v1.31.0-rc.1](#v1310-rc1) - - [Downloads for v1.31.0-rc.1](#downloads-for-v1310-rc1) +- [v1.31.0](#v1310) + - [Downloads for v1.31.0](#downloads-for-v1310) - [Source Code](#source-code-10) - [Client Binaries](#client-binaries-10) - [Server Binaries](#server-binaries-10) - [Node Binaries](#node-binaries-10) - [Container Images](#container-images-10) - - [Changelog since v1.31.0-rc.0](#changelog-since-v1310-rc0) + - [Changelog since v1.30.0](#changelog-since-v1300) + - [Urgent Upgrade Notes](#urgent-upgrade-notes) + - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade) - [Changes by Kind](#changes-by-kind-10) - - [API Change](#api-change-3) + - [Deprecation](#deprecation-1) + - [API Change](#api-change-2) - [Feature](#feature-7) + - [Failing Test](#failing-test) - [Bug or Regression](#bug-or-regression-10) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-5) - [Dependencies](#dependencies-10) - [Added](#added-10) - [Changed](#changed-10) - [Removed](#removed-10) -- [v1.31.0-rc.0](#v1310-rc0) - - [Downloads for v1.31.0-rc.0](#downloads-for-v1310-rc0) +- [v1.31.0-rc.1](#v1310-rc1) + - [Downloads for v1.31.0-rc.1](#downloads-for-v1310-rc1) - [Source Code](#source-code-11) - [Client Binaries](#client-binaries-11) - [Server Binaries](#server-binaries-11) - [Node Binaries](#node-binaries-11) - [Container Images](#container-images-11) - - [Changelog since v1.31.0-beta.0](#changelog-since-v1310-beta0) - - [Urgent Upgrade Notes](#urgent-upgrade-notes-1) - - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-1) + - [Changelog since v1.31.0-rc.0](#changelog-since-v1310-rc0) - [Changes by Kind](#changes-by-kind-11) - - [Deprecation](#deprecation-2) - - [API Change](#api-change-4) + - [API Change](#api-change-3) - [Feature](#feature-8) - - [Failing Test](#failing-test-1) - [Bug or Regression](#bug-or-regression-11) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-5) - [Dependencies](#dependencies-11) - [Added](#added-11) - [Changed](#changed-11) - [Removed](#removed-11) -- [v1.31.0-beta.0](#v1310-beta0) - - [Downloads for v1.31.0-beta.0](#downloads-for-v1310-beta0) +- [v1.31.0-rc.0](#v1310-rc0) + - [Downloads for v1.31.0-rc.0](#downloads-for-v1310-rc0) - [Source Code](#source-code-12) - [Client Binaries](#client-binaries-12) - [Server Binaries](#server-binaries-12) - [Node Binaries](#node-binaries-12) - [Container Images](#container-images-12) - - [Changelog since v1.31.0-alpha.3](#changelog-since-v1310-alpha3) + - [Changelog since v1.31.0-beta.0](#changelog-since-v1310-beta0) + - [Urgent Upgrade Notes](#urgent-upgrade-notes-1) + - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-1) - [Changes by Kind](#changes-by-kind-12) - - [API Change](#api-change-5) + - [Deprecation](#deprecation-2) + - [API Change](#api-change-4) - [Feature](#feature-9) + - [Failing Test](#failing-test-1) - [Bug or Regression](#bug-or-regression-12) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-6) - [Dependencies](#dependencies-12) - [Added](#added-12) - [Changed](#changed-12) - [Removed](#removed-12) -- [v1.31.0-alpha.3](#v1310-alpha3) - - [Downloads for v1.31.0-alpha.3](#downloads-for-v1310-alpha3) +- [v1.31.0-beta.0](#v1310-beta0) + - [Downloads for v1.31.0-beta.0](#downloads-for-v1310-beta0) - [Source Code](#source-code-13) - [Client Binaries](#client-binaries-13) - [Server Binaries](#server-binaries-13) - [Node Binaries](#node-binaries-13) - [Container Images](#container-images-13) - - [Changelog since v1.31.0-alpha.2](#changelog-since-v1310-alpha2) + - [Changelog since v1.31.0-alpha.3](#changelog-since-v1310-alpha3) - [Changes by Kind](#changes-by-kind-13) - - [API Change](#api-change-6) + - [API Change](#api-change-5) - [Feature](#feature-10) - [Bug or Regression](#bug-or-regression-13) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-7) @@ -236,50 +235,161 @@ - [Added](#added-13) - [Changed](#changed-13) - [Removed](#removed-13) -- [v1.31.0-alpha.2](#v1310-alpha2) - - [Downloads for v1.31.0-alpha.2](#downloads-for-v1310-alpha2) +- [v1.31.0-alpha.3](#v1310-alpha3) + - [Downloads for v1.31.0-alpha.3](#downloads-for-v1310-alpha3) - [Source Code](#source-code-14) - [Client Binaries](#client-binaries-14) - [Server Binaries](#server-binaries-14) - [Node Binaries](#node-binaries-14) - [Container Images](#container-images-14) - - [Changelog since v1.31.0-alpha.1](#changelog-since-v1310-alpha1) - - [Urgent Upgrade Notes](#urgent-upgrade-notes-2) - - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-2) + - [Changelog since v1.31.0-alpha.2](#changelog-since-v1310-alpha2) - [Changes by Kind](#changes-by-kind-14) - - [API Change](#api-change-7) + - [API Change](#api-change-6) - [Feature](#feature-11) - - [Failing Test](#failing-test-2) - [Bug or Regression](#bug-or-regression-14) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-8) - [Dependencies](#dependencies-14) - [Added](#added-14) - [Changed](#changed-14) - [Removed](#removed-14) -- [v1.31.0-alpha.1](#v1310-alpha1) - - [Downloads for v1.31.0-alpha.1](#downloads-for-v1310-alpha1) +- [v1.31.0-alpha.2](#v1310-alpha2) + - [Downloads for v1.31.0-alpha.2](#downloads-for-v1310-alpha2) - [Source Code](#source-code-15) - [Client Binaries](#client-binaries-15) - [Server Binaries](#server-binaries-15) - [Node Binaries](#node-binaries-15) - [Container Images](#container-images-15) - - [Changelog since v1.30.0](#changelog-since-v1300-1) - - [Urgent Upgrade Notes](#urgent-upgrade-notes-3) - - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-3) + - [Changelog since v1.31.0-alpha.1](#changelog-since-v1310-alpha1) + - [Urgent Upgrade Notes](#urgent-upgrade-notes-2) + - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-2) - [Changes by Kind](#changes-by-kind-15) - - [Deprecation](#deprecation-3) - - [API Change](#api-change-8) + - [API Change](#api-change-7) - [Feature](#feature-12) - - [Failing Test](#failing-test-3) + - [Failing Test](#failing-test-2) - [Bug or Regression](#bug-or-regression-15) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-9) - [Dependencies](#dependencies-15) - [Added](#added-15) - [Changed](#changed-15) - [Removed](#removed-15) +- [v1.31.0-alpha.1](#v1310-alpha1) + - [Downloads for v1.31.0-alpha.1](#downloads-for-v1310-alpha1) + - [Source Code](#source-code-16) + - [Client Binaries](#client-binaries-16) + - [Server Binaries](#server-binaries-16) + - [Node Binaries](#node-binaries-16) + - [Container Images](#container-images-16) + - [Changelog since v1.30.0](#changelog-since-v1300-1) + - [Urgent Upgrade Notes](#urgent-upgrade-notes-3) + - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade-3) + - [Changes by Kind](#changes-by-kind-16) + - [Deprecation](#deprecation-3) + - [API Change](#api-change-8) + - [Feature](#feature-13) + - [Failing Test](#failing-test-3) + - [Bug or Regression](#bug-or-regression-16) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-10) + - [Dependencies](#dependencies-16) + - [Added](#added-16) + - [Changed](#changed-16) + - [Removed](#removed-16) +# v1.31.10 + + +## Downloads for v1.31.10 + + + +### Source Code + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes.tar.gz) | e42497bc8056b9c71fd81a92905b8892356ce04acd592e9007179dfd0bbaac1bbe781a4bd2c52324c9cbf8d6638dc4dff666f4b8f2d7117b110916891a96950c +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-src.tar.gz) | a35b67d29628848c6c9c728f5b6620a50d1a90717566860571651d6c890fb3d2ac460237597bb3c302892c59f8fcc832b261248314265fc6c86748a3b9fceef4 + +### Client Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-darwin-amd64.tar.gz) | e2c0c34069f5d6306ae6ca800e2bd57dc44df3b68809c2bf04f88c179450f50a5cbe47788e618543ba6a2fef96d290ab59556e1c88db6762b3e8528e1fa2b7da +[kubernetes-client-darwin-arm64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-darwin-arm64.tar.gz) | 91074e58286f5275b889bcfead31166d6c7cb4e1b6020b33bc1642e64937be672fd4ddee82f1b4bd4e24851d6682390d4ff7b5742f27bbe9675570800328e038 +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-linux-386.tar.gz) | 089b78addc1e248f8c846e89e77bbda06145c37d27dae762c66691bc3bdcb9411af545d04b61d234c4bf075880872661a2da5871b9439d5f59dcdbd2520ec705 +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-linux-amd64.tar.gz) | 396a10e90306d4df08708497c26ab23b5aba23b3a86656bcdf181e2af969b7a16a11f2f19fd1b31a314fb4c3cd039242f6b7ab0a89c3231be520582e55172f00 +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-linux-arm.tar.gz) | c7ecb36da06f731862dc770e8fae4f64dccac81b62c1e0d6f06af69698b891396dd46d1d3cda3026ecfcb062c76675cc79f82e24ab277a7be39288b23f6b859d +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-linux-arm64.tar.gz) | 545f6d33b1a5324d9eabc5c7d69d09a3c9cc24dc2761373424b0049b6c404cf6700bd0071cc0d7755ac1eb582d8a9dc65b6e66fe95fdd53556bb819b66a2a703 +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-linux-ppc64le.tar.gz) | c17d9390a264a1387e216066c5807e8925a02e6e686f3180639023ec6b0c076e4e2fbb9cd20afbe23da214d6117aa949590a20f5dec84d19fbb796ba64d09b5d +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-linux-s390x.tar.gz) | 8676a9b9ed62d87a952d98b032151b5d759460e619108c9934f348001774c481cba66fe403f5b751fe4f017fa81a393f16b2952197c56676274d674abc16f9e5 +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-windows-386.tar.gz) | 6dc0ac7a6c907e56295ace07a60315310fe0a07688846471f4e7fb4b8024eb752f44f818797dcf4da62ca6e7aee812d226def6dced113fb477eb3a8c1318f9f9 +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-windows-amd64.tar.gz) | de97ea0d08a90a2af5e40fdc1b6600bbf125fffb4c2bb26aabeac26216f157d328a784015aa84537ae1d150b68639e6bac9848edcba668a0e0c24f66f86ea7a0 +[kubernetes-client-windows-arm64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-client-windows-arm64.tar.gz) | f39a70aff8e10807201bef71e1352ce7fe3e112424d3480fac081b9c9a4c03707ed3616fdf5e419275984e34be98d43344c822040f7c6e818188f5e6c7f4e57f + +### Server Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-server-linux-amd64.tar.gz) | 078992297cf07412d121ed13680ba7f96039543d06c57589d7b48b6c832a3cf50e0b2368eaa77ba6f6ada690434493e98ca629ceb72e43dd1acfe2667abe1fba +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-server-linux-arm64.tar.gz) | c9c9299b1a7e85f1de173d70541797096972570265bab307ad21a0527dadec7e61eb6b151ff32767a61b6361a5a5a8281702dbd46fd3dd5afdb09a78eb0223cb +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-server-linux-ppc64le.tar.gz) | ab94b4bf42fb2ce73befd7e7d0d467c31e3d70df61b47cab95702ce19e6e8b778e2f9f396c81b6f61a4631fe84b7646d1e85b840380298536e66b7a848c1ac29 +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-server-linux-s390x.tar.gz) | bda80644f8dc143d60f9ba2138a96c953501a1b46e25ca3602c456ee076462b305c24a57852053fa13ec7ac27d60332d33ad50e90bd13114930532b4a5644bf6 + +### Node Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-node-linux-amd64.tar.gz) | e8aab946fdf6e0d84b3dada524d7eb2537b2c21e1fe8b2e7d2dab991d3b6b43ea87d9c4f492b2f92c441297aec30302573aa84fc2d5d3c854a89a0ae82dfca9f +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-node-linux-arm64.tar.gz) | dfd450e2ac363c2d2aeefa82d77ffdea41d3c7593427ed42c016025a0fd16730ecc54d7b7d0a32225453a5be544893c7105cbab570f3aa4626872d44e550d4b7 +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-node-linux-ppc64le.tar.gz) | eed62687cfa85c0ec943013f033976b87818a7ad7fc883c88e5268c9f24c914242699eaf57218cf8f1ed9ebbf0432461148b7a1281a544cdb96632298dd45201 +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-node-linux-s390x.tar.gz) | 709396599409c32b56a7cedf336663fd8e0b9634e36c463169a74a698a7b70c8bdb56aee7c7391a6876eb551c90c90ad8c719c31454c17796694f79605551712 +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.31.10/kubernetes-node-windows-amd64.tar.gz) | e347e42b6a2766c1adc4224d8e80fa87f2723044466fc96bb5a4cdea0695e819a61a758ef2455f12dd61f94564c9fc438b9e358bc021df763397b1f6948e4f4e + +### Container Images + +All container images are available as manifest lists and support the described +architectures. It is also possible to pull a specific architecture directly by +adding the "-$ARCH" suffix to the container image name. + +name | architectures +---- | ------------- +[registry.k8s.io/conformance:v1.31.10](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-s390x) +[registry.k8s.io/kube-apiserver:v1.31.10](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-s390x) +[registry.k8s.io/kube-controller-manager:v1.31.10](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-s390x) +[registry.k8s.io/kube-proxy:v1.31.10](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-s390x) +[registry.k8s.io/kube-scheduler:v1.31.10](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-s390x) +[registry.k8s.io/kubectl:v1.31.10](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-s390x) + +## Changelog since v1.31.9 + +## Changes by Kind + +### Feature + +- Kubernetes is now built using Go 1.23.10 ([#132224](https://github.com/kubernetes/kubernetes/pull/132224), [@cpanato](https://github.com/cpanato)) [SIG Release and Testing] +- Kubernetes is now built using Go 1.23.9 ([#131937](https://github.com/kubernetes/kubernetes/pull/131937), [@cpanato](https://github.com/cpanato)) [SIG Release and Testing] + +### Bug or Regression + +- Removed a warning around Linux user namespaces and kernel version. If the feature gate `UserNamespacesSupport` was enabled, the kubelet previously warned when detecting a Linux kernel version earlier than 6.3.0. User namespace support on Linux typically does still need kernel 6.3 or newer, but it can work in older kernels too. ([#131785](https://github.com/kubernetes/kubernetes/pull/131785), [@rata](https://github.com/rata)) [SIG Node] + +### Other (Cleanup or Flake) + +- Improve error message when a pod with user namespaces is created and the runtime doesn't support user namespaces. ([#131783](https://github.com/kubernetes/kubernetes/pull/131783), [@rata](https://github.com/rata)) [SIG Node] + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + + # v1.31.9 From fc02d1eaf52b0b34b0d5fcc0eb930ffb930e85ef Mon Sep 17 00:00:00 2001 From: Ben Luddy Date: Tue, 24 Jun 2025 13:06:21 -0400 Subject: [PATCH 4/6] Use per-policy marker names for VAP integration tests. Writes to policy resources don't instantaneously take effect in admission. ValidatingAdmissionPolicy integration tests determine that the policies under test have taken effect by adding a sentinel policy rule and polling until that rule is applied to a request. If the marker resource names are the same for each test case in a series of test cases, then observing a policy's effect on a marker request only indicates that _any_ test policy is in effect, but it's not necessarily the policy the current test case is waiting for. For example: 1. Test 1 creates a policy and binding. 2. The policy and binding are observed by the admission plugin and take effect. 3. Test 1 observes that a policy is in effect via marker requests. 4. Test 1 exercises the behavior under test and successfully deletes the policy and binding it created. 5. Test 2 creates a policy and binding. 6. Test 2 observes that a policy is in effect via marker requests, but the policy in effect is still the one created by Test 1. 7. Test 2 exercises the behavior under test, which fails because it was evaluated against Test 1's policy. Generating a per-policy name for the marker resource in each test resolves the timing issue. In the example, step (6) will not proceed until the admission plugin has observed the policy and binding created in (5). --- .../cel/validatingadmissionpolicy_test.go | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/test/integration/apiserver/cel/validatingadmissionpolicy_test.go b/test/integration/apiserver/cel/validatingadmissionpolicy_test.go index c27cc1754feda..1a433fc212c4f 100644 --- a/test/integration/apiserver/cel/validatingadmissionpolicy_test.go +++ b/test/integration/apiserver/cel/validatingadmissionpolicy_test.go @@ -64,6 +64,8 @@ import ( authorizationv1 "k8s.io/api/authorization/v1" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + utilrand "k8s.io/apimachinery/pkg/util/rand" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" ) // Test_ValidateNamespace_NoParams tests a ValidatingAdmissionPolicy that validates creation of a Namespace with no params. @@ -2870,8 +2872,17 @@ func serviceAccountClient(namespace, name string) clientFn { func withWaitReadyConstraintAndExpression(policy *admissionregistrationv1.ValidatingAdmissionPolicy) *admissionregistrationv1.ValidatingAdmissionPolicy { policy = policy.DeepCopy() + + testMarkerName := fmt.Sprintf("test-marker-%s", utilrand.String(utilvalidation.DNS1123SubdomainMaxLength-len("test-marker-"))) + annotations := policy.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations["test-marker-name"] = testMarkerName + policy.SetAnnotations(annotations) + policy.Spec.MatchConstraints.ResourceRules = append(policy.Spec.MatchConstraints.ResourceRules, admissionregistrationv1.NamedRuleWithOperations{ - ResourceNames: []string{"test-marker"}, + ResourceNames: []string{testMarkerName}, RuleWithOperations: admissionregistrationv1.RuleWithOperations{ Operations: []admissionregistrationv1.OperationType{ "UPDATE", @@ -2890,7 +2901,7 @@ func withWaitReadyConstraintAndExpression(policy *admissionregistrationv1.Valida }, }) policy.Spec.Validations = append([]admissionregistrationv1.Validation{{ - Expression: "object.metadata.name != 'test-marker'", + Expression: fmt.Sprintf("object.metadata.name != '%s'", testMarkerName), Message: "marker denied; policy is ready", }}, policy.Spec.Validations...) return policy @@ -2905,14 +2916,23 @@ func createAndWaitReadyNamespaced(t *testing.T, client clientset.Interface, bind } func createAndWaitReadyNamespacedWithWarnHandler(t *testing.T, client clientset.Interface, binding *admissionregistrationv1.ValidatingAdmissionPolicyBinding, matchLabels map[string]string, ns string, handler *warningHandler) error { - marker := &v1.Endpoints{ObjectMeta: metav1.ObjectMeta{Name: "test-marker", Namespace: ns, Labels: matchLabels}} + policy, err := client.AdmissionregistrationV1().ValidatingAdmissionPolicies().Get(context.TODO(), binding.Spec.PolicyName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + testMarkerName := "test-marker" + if testMarkerNameAnnotation, ok := policy.GetAnnotations()["test-marker-name"]; ok { + testMarkerName = testMarkerNameAnnotation + } + + marker := &v1.Endpoints{ObjectMeta: metav1.ObjectMeta{Name: testMarkerName, Namespace: ns, Labels: matchLabels}} defer func() { err := client.CoreV1().Endpoints(ns).Delete(context.TODO(), marker.Name, metav1.DeleteOptions{}) if err != nil { t.Logf("error deleting marker: %v", err) } }() - marker, err := client.CoreV1().Endpoints(ns).Create(context.TODO(), marker, metav1.CreateOptions{}) + marker, err = client.CoreV1().Endpoints(ns).Create(context.TODO(), marker, metav1.CreateOptions{}) if err != nil { return err } From 7e94e1ce5c71407a782db8ef701e290bb0246da0 Mon Sep 17 00:00:00 2001 From: Kubernetes Release Robot Date: Tue, 15 Jul 2025 18:05:31 +0000 Subject: [PATCH 5/6] Release commit for Kubernetes v1.31.11 From 14ddfb48cd57d4a10044cc2f69b481ed81f39ba6 Mon Sep 17 00:00:00 2001 From: Allen Ray Date: Thu, 24 Jul 2025 12:57:47 -0400 Subject: [PATCH 6/6] UPSTREAM: : hack/update-vendor.sh, make update and update image --- openshift-hack/images/hyperkube/Dockerfile.rhel | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openshift-hack/images/hyperkube/Dockerfile.rhel b/openshift-hack/images/hyperkube/Dockerfile.rhel index d29df10228c2e..f5dac3e825e94 100644 --- a/openshift-hack/images/hyperkube/Dockerfile.rhel +++ b/openshift-hack/images/hyperkube/Dockerfile.rhel @@ -12,6 +12,6 @@ FROM registry.ci.openshift.org/ocp/4.18:base-rhel9 RUN yum install -y --setopt=tsflags=nodocs --setopt=skip_missing_names_on_install=False iproute && yum clean all COPY --from=builder /tmp/build/* /usr/bin/ LABEL io.k8s.display-name="OpenShift Kubernetes Server Commands" \ - io.k8s.description="OpenShift is a platform for developing, building, and deploying containerized applications." \ - io.openshift.tags="openshift,hyperkube" \ - io.openshift.build.versions="kubernetes=1.31.10" + io.k8s.description="OpenShift is a platform for developing, building, and deploying containerized applications." \ + io.openshift.tags="openshift,hyperkube" \ + io.openshift.build.versions="kubernetes=1.31.11"