From d5df907c3cb6fa3a6a95fa3d12d92fe37e2f79e9 Mon Sep 17 00:00:00 2001 From: Kubernetes Release Robot Date: Tue, 22 Apr 2025 16:22:00 +0000 Subject: [PATCH 01/11] Update CHANGELOG/CHANGELOG-1.32.md for v1.32.4 --- CHANGELOG/CHANGELOG-1.32.md | 251 +++++++++++++++++++++++++----------- 1 file changed, 176 insertions(+), 75 deletions(-) diff --git a/CHANGELOG/CHANGELOG-1.32.md b/CHANGELOG/CHANGELOG-1.32.md index 1a53cbc33bbee..1c6bc92ef0ae7 100644 --- a/CHANGELOG/CHANGELOG-1.32.md +++ b/CHANGELOG/CHANGELOG-1.32.md @@ -1,200 +1,301 @@ -- [v1.32.3](#v1323) - - [Downloads for v1.32.3](#downloads-for-v1323) +- [v1.32.4](#v1324) + - [Downloads for v1.32.4](#downloads-for-v1324) - [Source Code](#source-code) - [Client Binaries](#client-binaries) - [Server Binaries](#server-binaries) - [Node Binaries](#node-binaries) - [Container Images](#container-images) - - [Changelog since v1.32.2](#changelog-since-v1322) + - [Changelog since v1.32.3](#changelog-since-v1323) - [Changes by Kind](#changes-by-kind) - - [API Change](#api-change) - [Bug or Regression](#bug-or-regression) - [Dependencies](#dependencies) - [Added](#added) - [Changed](#changed) - [Removed](#removed) -- [v1.32.2](#v1322) - - [Downloads for v1.32.2](#downloads-for-v1322) +- [v1.32.3](#v1323) + - [Downloads for v1.32.3](#downloads-for-v1323) - [Source Code](#source-code-1) - [Client Binaries](#client-binaries-1) - [Server Binaries](#server-binaries-1) - [Node Binaries](#node-binaries-1) - [Container Images](#container-images-1) - - [Changelog since v1.32.1](#changelog-since-v1321) - - [Important Security Information](#important-security-information) - - [CVE-2025-0426: Node Denial of Service via Kubelet Checkpoint API](#cve-2025-0426-node-denial-of-service-via-kubelet-checkpoint-api) + - [Changelog since v1.32.2](#changelog-since-v1322) - [Changes by Kind](#changes-by-kind-1) - - [Feature](#feature) + - [API Change](#api-change) - [Bug or Regression](#bug-or-regression-1) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake) - [Dependencies](#dependencies-1) - [Added](#added-1) - [Changed](#changed-1) - [Removed](#removed-1) -- [v1.32.1](#v1321) - - [Downloads for v1.32.1](#downloads-for-v1321) +- [v1.32.2](#v1322) + - [Downloads for v1.32.2](#downloads-for-v1322) - [Source Code](#source-code-2) - [Client Binaries](#client-binaries-2) - [Server Binaries](#server-binaries-2) - [Node Binaries](#node-binaries-2) - [Container Images](#container-images-2) - - [Changelog since v1.32.0](#changelog-since-v1320) - - [Important Security Information](#important-security-information-1) - - [CVE-2024-9042: Command Injection affecting Windows nodes via nodes/*/logs/query API](#cve-2024-9042-command-injection-affecting-windows-nodes-via-nodeslogsquery-api) + - [Changelog since v1.32.1](#changelog-since-v1321) + - [Important Security Information](#important-security-information) + - [CVE-2025-0426: Node Denial of Service via Kubelet Checkpoint API](#cve-2025-0426-node-denial-of-service-via-kubelet-checkpoint-api) - [Changes by Kind](#changes-by-kind-2) - - [API Change](#api-change-1) - - [Feature](#feature-1) + - [Feature](#feature) - [Bug or Regression](#bug-or-regression-2) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake) - [Dependencies](#dependencies-2) - [Added](#added-2) - [Changed](#changed-2) - [Removed](#removed-2) -- [v1.32.0](#v1320) - - [Downloads for v1.32.0](#downloads-for-v1320) +- [v1.32.1](#v1321) + - [Downloads for v1.32.1](#downloads-for-v1321) - [Source Code](#source-code-3) - [Client Binaries](#client-binaries-3) - [Server Binaries](#server-binaries-3) - [Node Binaries](#node-binaries-3) - [Container Images](#container-images-3) - - [Changelog since v1.31.0](#changelog-since-v1310) - - [Urgent Upgrade Notes](#urgent-upgrade-notes) + - [Changelog since v1.32.0](#changelog-since-v1320) + - [Important Security Information](#important-security-information-1) + - [CVE-2024-9042: Command Injection affecting Windows nodes via nodes/*/logs/query API](#cve-2024-9042-command-injection-affecting-windows-nodes-via-nodeslogsquery-api) - [Changes by Kind](#changes-by-kind-3) - - [Deprecation](#deprecation) - - [API Change](#api-change-2) - - [Feature](#feature-2) - - [Documentation](#documentation) - - [Failing Test](#failing-test) + - [API Change](#api-change-1) + - [Feature](#feature-1) - [Bug or Regression](#bug-or-regression-3) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-1) - [Dependencies](#dependencies-3) - [Added](#added-3) - [Changed](#changed-3) - [Removed](#removed-3) -- [v1.32.0-rc.2](#v1320-rc2) - - [Downloads for v1.32.0-rc.2](#downloads-for-v1320-rc2) +- [v1.32.0](#v1320) + - [Downloads for v1.32.0](#downloads-for-v1320) - [Source Code](#source-code-4) - [Client Binaries](#client-binaries-4) - [Server Binaries](#server-binaries-4) - [Node Binaries](#node-binaries-4) - [Container Images](#container-images-4) - - [Changelog since v1.32.0-rc.1](#changelog-since-v1320-rc1) + - [Changelog since v1.31.0](#changelog-since-v1310) + - [Urgent Upgrade Notes](#urgent-upgrade-notes) - [Changes by Kind](#changes-by-kind-4) - - [API Change](#api-change-3) + - [Deprecation](#deprecation) + - [API Change](#api-change-2) + - [Feature](#feature-2) + - [Documentation](#documentation) + - [Failing Test](#failing-test) - [Bug or Regression](#bug-or-regression-4) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-1) - [Dependencies](#dependencies-4) - [Added](#added-4) - [Changed](#changed-4) - [Removed](#removed-4) -- [v1.32.0-rc.1](#v1320-rc1) - - [Downloads for v1.32.0-rc.1](#downloads-for-v1320-rc1) +- [v1.32.0-rc.2](#v1320-rc2) + - [Downloads for v1.32.0-rc.2](#downloads-for-v1320-rc2) - [Source Code](#source-code-5) - [Client Binaries](#client-binaries-5) - [Server Binaries](#server-binaries-5) - [Node Binaries](#node-binaries-5) - [Container Images](#container-images-5) - - [Changelog since v1.32.0-rc.0](#changelog-since-v1320-rc0) + - [Changelog since v1.32.0-rc.1](#changelog-since-v1320-rc1) + - [Changes by Kind](#changes-by-kind-5) + - [API Change](#api-change-3) + - [Bug or Regression](#bug-or-regression-5) - [Dependencies](#dependencies-5) - [Added](#added-5) - [Changed](#changed-5) - [Removed](#removed-5) -- [v1.32.0-rc.0](#v1320-rc0) - - [Downloads for v1.32.0-rc.0](#downloads-for-v1320-rc0) +- [v1.32.0-rc.1](#v1320-rc1) + - [Downloads for v1.32.0-rc.1](#downloads-for-v1320-rc1) - [Source Code](#source-code-6) - [Client Binaries](#client-binaries-6) - [Server Binaries](#server-binaries-6) - [Node Binaries](#node-binaries-6) - [Container Images](#container-images-6) - - [Changelog since v1.32.0-beta.0](#changelog-since-v1320-beta0) - - [Changes by Kind](#changes-by-kind-5) - - [API Change](#api-change-4) - - [Feature](#feature-3) - - [Bug or Regression](#bug-or-regression-5) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-2) + - [Changelog since v1.32.0-rc.0](#changelog-since-v1320-rc0) - [Dependencies](#dependencies-6) - [Added](#added-6) - [Changed](#changed-6) - [Removed](#removed-6) -- [v1.32.0-beta.0](#v1320-beta0) - - [Downloads for v1.32.0-beta.0](#downloads-for-v1320-beta0) +- [v1.32.0-rc.0](#v1320-rc0) + - [Downloads for v1.32.0-rc.0](#downloads-for-v1320-rc0) - [Source Code](#source-code-7) - [Client Binaries](#client-binaries-7) - [Server Binaries](#server-binaries-7) - [Node Binaries](#node-binaries-7) - [Container Images](#container-images-7) - - [Changelog since v1.32.0-alpha.3](#changelog-since-v1320-alpha3) - - [Urgent Upgrade Notes](#urgent-upgrade-notes-1) - - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade) + - [Changelog since v1.32.0-beta.0](#changelog-since-v1320-beta0) - [Changes by Kind](#changes-by-kind-6) - - [Deprecation](#deprecation-1) - - [API Change](#api-change-5) - - [Feature](#feature-4) + - [API Change](#api-change-4) + - [Feature](#feature-3) - [Bug or Regression](#bug-or-regression-6) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-3) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-2) - [Dependencies](#dependencies-7) - [Added](#added-7) - [Changed](#changed-7) - [Removed](#removed-7) -- [v1.32.0-alpha.3](#v1320-alpha3) - - [Downloads for v1.32.0-alpha.3](#downloads-for-v1320-alpha3) +- [v1.32.0-beta.0](#v1320-beta0) + - [Downloads for v1.32.0-beta.0](#downloads-for-v1320-beta0) - [Source Code](#source-code-8) - [Client Binaries](#client-binaries-8) - [Server Binaries](#server-binaries-8) - [Node Binaries](#node-binaries-8) - [Container Images](#container-images-8) - - [Changelog since v1.32.0-alpha.2](#changelog-since-v1320-alpha2) + - [Changelog since v1.32.0-alpha.3](#changelog-since-v1320-alpha3) + - [Urgent Upgrade Notes](#urgent-upgrade-notes-1) + - [(No, really, you MUST read this before you upgrade)](#no-really-you-must-read-this-before-you-upgrade) - [Changes by Kind](#changes-by-kind-7) - - [API Change](#api-change-6) - - [Feature](#feature-5) - - [Documentation](#documentation-1) + - [Deprecation](#deprecation-1) + - [API Change](#api-change-5) + - [Feature](#feature-4) - [Bug or Regression](#bug-or-regression-7) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-4) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-3) - [Dependencies](#dependencies-8) - [Added](#added-8) - [Changed](#changed-8) - [Removed](#removed-8) -- [v1.32.0-alpha.2](#v1320-alpha2) - - [Downloads for v1.32.0-alpha.2](#downloads-for-v1320-alpha2) +- [v1.32.0-alpha.3](#v1320-alpha3) + - [Downloads for v1.32.0-alpha.3](#downloads-for-v1320-alpha3) - [Source Code](#source-code-9) - [Client Binaries](#client-binaries-9) - [Server Binaries](#server-binaries-9) - [Node Binaries](#node-binaries-9) - [Container Images](#container-images-9) - - [Changelog since v1.32.0-alpha.1](#changelog-since-v1320-alpha1) + - [Changelog since v1.32.0-alpha.2](#changelog-since-v1320-alpha2) - [Changes by Kind](#changes-by-kind-8) - - [API Change](#api-change-7) - - [Feature](#feature-6) - - [Documentation](#documentation-2) + - [API Change](#api-change-6) + - [Feature](#feature-5) + - [Documentation](#documentation-1) - [Bug or Regression](#bug-or-regression-8) - - [Other (Cleanup or Flake)](#other-cleanup-or-flake-5) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-4) - [Dependencies](#dependencies-9) - [Added](#added-9) - [Changed](#changed-9) - [Removed](#removed-9) -- [v1.32.0-alpha.1](#v1320-alpha1) - - [Downloads for v1.32.0-alpha.1](#downloads-for-v1320-alpha1) +- [v1.32.0-alpha.2](#v1320-alpha2) + - [Downloads for v1.32.0-alpha.2](#downloads-for-v1320-alpha2) - [Source Code](#source-code-10) - [Client Binaries](#client-binaries-10) - [Server Binaries](#server-binaries-10) - [Node Binaries](#node-binaries-10) - [Container Images](#container-images-10) - - [Changelog since v1.31.0](#changelog-since-v1310-1) + - [Changelog since v1.32.0-alpha.1](#changelog-since-v1320-alpha1) - [Changes by Kind](#changes-by-kind-9) + - [API Change](#api-change-7) + - [Feature](#feature-6) + - [Documentation](#documentation-2) + - [Bug or Regression](#bug-or-regression-9) + - [Other (Cleanup or Flake)](#other-cleanup-or-flake-5) + - [Dependencies](#dependencies-10) + - [Added](#added-10) + - [Changed](#changed-10) + - [Removed](#removed-10) +- [v1.32.0-alpha.1](#v1320-alpha1) + - [Downloads for v1.32.0-alpha.1](#downloads-for-v1320-alpha1) + - [Source Code](#source-code-11) + - [Client Binaries](#client-binaries-11) + - [Server Binaries](#server-binaries-11) + - [Node Binaries](#node-binaries-11) + - [Container Images](#container-images-11) + - [Changelog since v1.31.0](#changelog-since-v1310-1) + - [Changes by Kind](#changes-by-kind-10) - [Deprecation](#deprecation-2) - [API Change](#api-change-8) - [Feature](#feature-7) - [Documentation](#documentation-3) - [Failing Test](#failing-test-1) - - [Bug or Regression](#bug-or-regression-9) + - [Bug or Regression](#bug-or-regression-10) - [Other (Cleanup or Flake)](#other-cleanup-or-flake-6) - - [Dependencies](#dependencies-10) - - [Added](#added-10) - - [Changed](#changed-10) - - [Removed](#removed-10) + - [Dependencies](#dependencies-11) + - [Added](#added-11) + - [Changed](#changed-11) + - [Removed](#removed-11) +# v1.32.4 + + +## Downloads for v1.32.4 + + + +### Source Code + +filename | sha512 hash +-------- | ----------- +[kubernetes.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes.tar.gz) | 8fc0b0a408ba8cb2e3970f88503ef803c0c3def5f65741baa08d7c1adbfd8c31241929dc8dc14e4a0f22915b167ffe7bb0cba4eb6529d86bbc794dac6b3b505f +[kubernetes-src.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-src.tar.gz) | e7f6cda46a998384e7dee8e448a454f08a77ab26ceeb57429d9c1f50dc8be44ebcdfeb7d328076178d48fb455e3b5011809b0c165c2a61762ae9cffd32adc9e1 + +### Client Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-client-darwin-amd64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-darwin-amd64.tar.gz) | 5e7917e6c0bac8298974f85caf4b3590903c82ea0f58ae6d1408e1c6bd91198e34c2aee0850a0409c72caf41a0d1bb7c0adccd9a9860c86abcc908ae954a9bcc +[kubernetes-client-darwin-arm64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-darwin-arm64.tar.gz) | 036e80fb03e42f0899ea348d7f37e548930f15277516a2df0149139c91f2b5c0c4e6ece591c448c20582354d4fd39a9082208fde1a8156632628104a8fd62c01 +[kubernetes-client-linux-386.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-linux-386.tar.gz) | 5d6ede77acfa71d8952311fbbe5f86765e6ec399553a7319512c6d69cea68755543ef52e449341ac3c6bfb011149af866ac2c0d0eb6dd07692cb31c7fa6b1a1f +[kubernetes-client-linux-amd64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-linux-amd64.tar.gz) | 924bd0cdbef91caab04b5e9c31017c24d9d7c718f6db9e2c61d5c203d579c8f0c00ac7451bd3658d5cdf31d7a08c8ee5884511d8e961f0e9331d00b1f6f03bee +[kubernetes-client-linux-arm.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-linux-arm.tar.gz) | 8f49c5cd1d9d74a9b4dfa4cf3c95d7d11bd62d750f9199d75b9cab47bdc0189b74fb92374cbfc051bb09dbd3d728fe3da380f80799de417de76d860a0ffe5825 +[kubernetes-client-linux-arm64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-linux-arm64.tar.gz) | bf84363c16f72863e38d9d67194531aabafb6a82a20e3361354cc037964205557e8a39b62fa23b3c435c87f989838b6619980ea5c325c456e5cd5d47564d1644 +[kubernetes-client-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-linux-ppc64le.tar.gz) | 55b7a446276545575b5c0bab7d0e3935a555a6c2ca357d0c3b6b949e14a02200082aa20e1790632e3ca6290a2da0fa00a5cad9083862e1ad89a3c1b3a6d20009 +[kubernetes-client-linux-s390x.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-linux-s390x.tar.gz) | ef4095d064aa1088f7bdd313b01f0ee132cab865bd52edaf737493e0f1948cf6c85aaa6312eeca683a759a779c7df579c8fe9a33bcb3fd9474fd76d80181774d +[kubernetes-client-windows-386.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-windows-386.tar.gz) | 23023d9cd2cfa3a2ee7a21e449daacf67225e66e178c11544bd40d7dd49ae4fa1c6ec486cf03cc4f393fc76244fdcd1c78dd63dccfab6c4b7f8bf0ed6c1558a6 +[kubernetes-client-windows-amd64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-windows-amd64.tar.gz) | 98493d754782c3de5d23901b6455f0d1440977de85ff77c492d623ee24891cce518edc14bfcc40432ad7ad03953f2a9b20a86eeb0b35ce125a21b724e30e305f +[kubernetes-client-windows-arm64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-client-windows-arm64.tar.gz) | 0ab5a1e05f29b50a6ae5dd1119f2e83b89a49a3a2f1a207fcb8d09758375090822e3a2f377065ef9468fbe4be784e10d5cd475f347f7dabd5d4d9a604edc05de + +### Server Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-server-linux-amd64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-server-linux-amd64.tar.gz) | 06c42d365aa4336881c81893d415a9f2be61857f9db36425e2a6d58fb016b4c1dbe2c51b98848adbbedb0f624f6648d1e93f65b2c94224d683f679754b108409 +[kubernetes-server-linux-arm64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-server-linux-arm64.tar.gz) | 96f92b8e619184f4f92af2aa7e1c6e992aca816c4bdefde28cccde0ea7693f9ec0ce2ada622183dac22273e050349027a753d1cf5a97c48a52c01b9672b5e503 +[kubernetes-server-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-server-linux-ppc64le.tar.gz) | 305a63907a071c10abf4a04ec73405793eb2abf1fbbf30173cbc787da75256d30b79531b4e0edfd3c8b160830f7d702c6942339159535add972427229faaab2f +[kubernetes-server-linux-s390x.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-server-linux-s390x.tar.gz) | 8c6ddff57a1bf721b53c2d2084c00278e2cbe7fb94dc94ae8aa37a81e8c9cf7c362cece984ca67b580abedc9905575790f919a28370bdeaa324721da01baa807 + +### Node Binaries + +filename | sha512 hash +-------- | ----------- +[kubernetes-node-linux-amd64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-node-linux-amd64.tar.gz) | f4c39d9dd27976bcb53bee3665a451cc0dbc6f967d20b8d517c4ccf3f20828f5fb7d9c84a458031f63cb1991997a1024a3a428f6a2bf6034007f62fe81bf0550 +[kubernetes-node-linux-arm64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-node-linux-arm64.tar.gz) | af79eb56591625fdff178a8625153162fd1a187b80c61f03a27010377204dc0d4101219bbb47ecb79f362c6d45514994a6579881bab01ee12e41cd2f0aa461da +[kubernetes-node-linux-ppc64le.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-node-linux-ppc64le.tar.gz) | b8b3398907cb64e8787ad8d1fe2f2176d1e0749bd5f673f15341c086d7b837529a3fd96e44a0152bda73a382f9089a432b370ab39f9565640c3a40babc0bd1be +[kubernetes-node-linux-s390x.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-node-linux-s390x.tar.gz) | b579dd86801d692c9930255353c005311d71b5c6b6678a1647df2b2dd4763f72001113e4391ebf14306791b468d3bb9bf6768eef63b4fab58de99cff311e5607 +[kubernetes-node-windows-amd64.tar.gz](https://dl.k8s.io/v1.32.4/kubernetes-node-windows-amd64.tar.gz) | 26367bbc47524601f6d81aaceb3ddcbea881b6f09edf8185f7748a8ff8563d66d290ef5105e795bf8f905d52b56a5ea224c3fc697988a7dc4424cf633cb88c3d + +### Container Images + +All container images are available as manifest lists and support the described +architectures. It is also possible to pull a specific architecture directly by +adding the "-$ARCH" suffix to the container image name. + +name | architectures +---- | ------------- +[registry.k8s.io/conformance:v1.32.4](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/conformance-s390x) +[registry.k8s.io/kube-apiserver:v1.32.4](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-apiserver-s390x) +[registry.k8s.io/kube-controller-manager:v1.32.4](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-controller-manager-s390x) +[registry.k8s.io/kube-proxy:v1.32.4](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-proxy-s390x) +[registry.k8s.io/kube-scheduler:v1.32.4](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kube-scheduler-s390x) +[registry.k8s.io/kubectl:v1.32.4](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl) | [amd64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-amd64), [arm64](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-arm64), [ppc64le](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-ppc64le), [s390x](https://console.cloud.google.com/artifacts/docker/k8s-artifacts-prod/southamerica-east1/images/kubectl-s390x) + +## Changelog since v1.32.3 + +## Changes by Kind + +### Bug or Regression + +- Fix a bug where kube-apiserver could emit an further watch even even if decryption failed for earlier event and it was not emitted. ([#131159](https://github.com/kubernetes/kubernetes/pull/131159), [@wojtek-t](https://github.com/wojtek-t)) [SIG API Machinery and Etcd] +- Fix kubelet restart unmounts volumes of running pods if the referenced PVC is being deleted by the user ([#130684](https://github.com/kubernetes/kubernetes/pull/130684), [@carlory](https://github.com/carlory)) [SIG Node, Storage and Testing] +- Fixes an issue in the CEL CIDR library where subnets contained within another CIDR were incorrectly rejected as not contained ([#130773](https://github.com/kubernetes/kubernetes/pull/130773), [@JoelSpeed](https://github.com/JoelSpeed)) [SIG API Machinery] + +## Dependencies + +### Added +_Nothing has changed._ + +### Changed +_Nothing has changed._ + +### Removed +_Nothing has changed._ + + + # v1.32.3 @@ -2743,4 +2844,4 @@ name | architectures - gopkg.in/errgo.v2: v2.1.0 - gopkg.in/ini.v1: v1.51.0 - gopkg.in/resty.v1: v1.12.0 -- rsc.io/binaryregexp: v0.2.0 +- rsc.io/binaryregexp: v0.2.0 \ No newline at end of file From 76bf5dbdc886c964926c8c2149bcc78a53471434 Mon Sep 17 00:00:00 2001 From: Prince Pereira Date: Tue, 1 Apr 2025 04:39:47 -0700 Subject: [PATCH 02/11] Fix for HNS local endpoint was being deleted instead of the remote endpoint. --- pkg/proxy/winkernel/hns.go | 65 +++++++++++++++++---------------- pkg/proxy/winkernel/hns_test.go | 64 ++++++++++++++++++++++++++++++++ pkg/proxy/winkernel/proxier.go | 15 ++++++-- 3 files changed, 110 insertions(+), 34 deletions(-) diff --git a/pkg/proxy/winkernel/hns.go b/pkg/proxy/winkernel/hns.go index 0a34ca07bf189..8f3498800f4f0 100644 --- a/pkg/proxy/winkernel/hns.go +++ b/pkg/proxy/winkernel/hns.go @@ -133,40 +133,43 @@ func (hns hns) getAllEndpointsByNetwork(networkName string) (map[string]*(endpoi continue } - // Add to map with key endpoint ID or IP address - // Storing this is expensive in terms of memory, however there is a bug in Windows Server 2019 that can cause two endpoints to be created with the same IP address. - // TODO: Store by IP only and remove any lookups by endpoint ID. - endpointInfos[ep.Id] = &endpointInfo{ - ip: ep.IpConfigurations[0].IpAddress, - isLocal: uint32(ep.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0, - macAddress: ep.MacAddress, - hnsID: ep.Id, - hns: hns, - // only ready and not terminating endpoints were added to HNS - ready: true, - serving: true, - terminating: false, - } - endpointInfos[ep.IpConfigurations[0].IpAddress] = endpointInfos[ep.Id] + for index, ipConfig := range ep.IpConfigurations { + + if index > 1 { + // Expecting only ipv4 and ipv6 ipaddresses + // This is highly unlikely to happen, but if it does, we should log a warning + // and break out of the loop + klog.Warning("Endpoint ipconfiguration holds more than 2 IP addresses.", "hnsID", ep.Id, "IP", ipConfig.IpAddress, "ipConfigCount", len(ep.IpConfigurations)) + break + } - if len(ep.IpConfigurations) == 1 { - continue - } + isLocal := uint32(ep.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0 - // If ipFamilyPolicy is RequireDualStack or PreferDualStack, then there will be 2 IPS (iPV4 and IPV6) - // in the endpoint list - endpointDualstack := &endpointInfo{ - ip: ep.IpConfigurations[1].IpAddress, - isLocal: uint32(ep.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0, - macAddress: ep.MacAddress, - hnsID: ep.Id, - hns: hns, - // only ready and not terminating endpoints were added to HNS - ready: true, - serving: true, - terminating: false, + if existingEp, ok := endpointInfos[ipConfig.IpAddress]; ok && isLocal { + // If the endpoint is already part of the queried endpoints map and is local, + // then we should not add it again to the map + // This is to avoid overwriting the remote endpoint info with a local endpoint. + klog.V(3).InfoS("Endpoint already exists in queried endpoints map; skipping.", "newLocalEndpoint", ep, "ipConfig", ipConfig, "existingEndpoint", existingEp) + continue + } + + // Add to map with key endpoint ID or IP address + // Storing this is expensive in terms of memory, however there is a bug in Windows Server 2019 and 2022 that can cause two endpoints (local and remote) to be created with the same IP address. + // TODO: Store by IP only and remove any lookups by endpoint ID. + epInfo := &endpointInfo{ + ip: ipConfig.IpAddress, + isLocal: isLocal, + macAddress: ep.MacAddress, + hnsID: ep.Id, + hns: hns, + // only ready and not terminating endpoints were added to HNS + ready: true, + serving: true, + terminating: false, + } + endpointInfos[ep.Id] = epInfo + endpointInfos[ipConfig.IpAddress] = epInfo } - endpointInfos[ep.IpConfigurations[1].IpAddress] = endpointDualstack } klog.V(3).InfoS("Queried endpoints from network", "network", networkName) klog.V(5).InfoS("Queried endpoints details", "network", networkName, "endpointInfos", endpointInfos) diff --git a/pkg/proxy/winkernel/hns_test.go b/pkg/proxy/winkernel/hns_test.go index 6e676016fd586..618b5aa125cb0 100644 --- a/pkg/proxy/winkernel/hns_test.go +++ b/pkg/proxy/winkernel/hns_test.go @@ -114,6 +114,70 @@ func TestGetAllEndpointsByNetwork(t *testing.T) { } } +func TestGetAllEndpointsByNetworkWithDupEP(t *testing.T) { + hcnMock := getHcnMock("L2Bridge") + hns := hns{hcn: hcnMock} + + ipv4Config := &hcn.IpConfig{ + IpAddress: epIpAddress, + } + ipv6Config := &hcn.IpConfig{ + IpAddress: epIpv6Address, + } + remoteEndpoint := &hcn.HostComputeEndpoint{ + IpConfigurations: []hcn.IpConfig{*ipv4Config, *ipv6Config}, + MacAddress: epMacAddress, + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 0, + }, + Flags: hcn.EndpointFlagsRemoteEndpoint, + } + Network, _ := hcnMock.GetNetworkByName(testNetwork) + remoteEndpoint, err := hns.hcn.CreateEndpoint(Network, remoteEndpoint) + if err != nil { + t.Error(err) + } + + // Create a duplicate local endpoint with the same IP address + dupLocalEndpoint := &hcn.HostComputeEndpoint{ + IpConfigurations: []hcn.IpConfig{*ipv4Config, *ipv6Config}, + MacAddress: epMacAddress, + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 0, + }, + } + + dupLocalEndpoint, err = hns.hcn.CreateEndpoint(Network, dupLocalEndpoint) + if err != nil { + t.Error(err) + } + + mapEndpointsInfo, err := hns.getAllEndpointsByNetwork(Network.Name) + if err != nil { + t.Error(err) + } + endpointIpv4, ipv4EpPresent := mapEndpointsInfo[ipv4Config.IpAddress] + assert.True(t, ipv4EpPresent, "IPV4 endpoint is missing in Dualstack mode") + assert.Equal(t, endpointIpv4.ip, epIpAddress, "IPV4 IP is missing in Dualstack mode") + assert.Equal(t, endpointIpv4.hnsID, remoteEndpoint.Id, "HNS ID is not matching with remote endpoint") + + endpointIpv6, ipv6EpPresent := mapEndpointsInfo[ipv6Config.IpAddress] + assert.True(t, ipv6EpPresent, "IPV6 endpoint is missing in Dualstack mode") + assert.Equal(t, endpointIpv6.ip, epIpv6Address, "IPV6 IP is missing in Dualstack mode") + assert.Equal(t, endpointIpv6.hnsID, remoteEndpoint.Id, "HNS ID is not matching with remote endpoint") + + err = hns.hcn.DeleteEndpoint(remoteEndpoint) + if err != nil { + t.Error(err) + } + err = hns.hcn.DeleteEndpoint(dupLocalEndpoint) + if err != nil { + t.Error(err) + } +} + func TestGetEndpointByID(t *testing.T) { // TODO: remove skip once the test has been fixed. t.Skip("Skipping failing test on Windows.") diff --git a/pkg/proxy/winkernel/proxier.go b/pkg/proxy/winkernel/proxier.go index 92a8969c98fdf..d523976e8afca 100644 --- a/pkg/proxy/winkernel/proxier.go +++ b/pkg/proxy/winkernel/proxier.go @@ -492,6 +492,11 @@ func (ep *endpointInfo) DecrementRefCount() { if !ep.IsLocal() && ep.refCount != nil && *ep.refCount > 0 { *ep.refCount-- } + refCount := 0 + if ep.refCount != nil { + refCount = int(*ep.refCount) + } + klog.V(5).InfoS("Endpoint RefCount after decrement.", "endpointInfo", ep, "refCount", refCount) } func (ep *endpointInfo) Cleanup() { @@ -1709,10 +1714,14 @@ func (proxier *Proxier) syncProxyRules() { // remove stale endpoint refcount entries for epIP := range proxier.terminatedEndpoints { - if epToDelete := queriedEndpoints[epIP]; epToDelete != nil && epToDelete.hnsID != "" { + klog.V(5).InfoS("Terminated endpoints ready for deletion", "epIP", epIP) + if epToDelete := queriedEndpoints[epIP]; epToDelete != nil && epToDelete.hnsID != "" && !epToDelete.IsLocal() { if refCount := proxier.endPointsRefCount.getRefCount(epToDelete.hnsID); refCount == nil || *refCount == 0 { - klog.V(3).InfoS("Deleting unreferenced remote endpoint", "hnsID", epToDelete.hnsID) - proxier.hns.deleteEndpoint(epToDelete.hnsID) + klog.V(3).InfoS("Deleting unreferenced remote endpoint", "hnsID", epToDelete.hnsID, "IP", epToDelete.ip) + err := proxier.hns.deleteEndpoint(epToDelete.hnsID) + if err != nil { + klog.ErrorS(err, "Deleting unreferenced remote endpoint failed", "hnsID", epToDelete.hnsID) + } } } } From 0bec23c95771c48bf7c07fec913582d373e6930c Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Wed, 23 Apr 2025 11:37:22 -0400 Subject: [PATCH 03/11] Check for newer fields when deciding expansion recovery feature status --- .../operationexecutor/operation_generator.go | 5 + .../operation_generator_test.go | 103 ++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/pkg/volume/util/operationexecutor/operation_generator.go b/pkg/volume/util/operationexecutor/operation_generator.go index 5523b61a5ec66..7b8a765583eff 100644 --- a/pkg/volume/util/operationexecutor/operation_generator.go +++ b/pkg/volume/util/operationexecutor/operation_generator.go @@ -2075,6 +2075,11 @@ func (og *operationGenerator) checkForRecoveryFromExpansion(pvc *v1.PersistentVo featureGateStatus := utilfeature.DefaultFeatureGate.Enabled(features.RecoverVolumeExpansionFailure) if !featureGateStatus { + // even though RecoverVolumeExpansionFailure feature-gate is disabled, we should consider it enabled + // if resizeStatus is not empty or allocatedresources is set + if resizeStatus != "" || allocatedResource != nil { + return true + } return false } diff --git a/pkg/volume/util/operationexecutor/operation_generator_test.go b/pkg/volume/util/operationexecutor/operation_generator_test.go index 7bf0dabd38b10..bf3c8e4e94e1b 100644 --- a/pkg/volume/util/operationexecutor/operation_generator_test.go +++ b/pkg/volume/util/operationexecutor/operation_generator_test.go @@ -402,6 +402,109 @@ func TestExpandDuringMount(t *testing.T) { }) } } +func TestCheckForRecoveryFromExpansion(t *testing.T) { + tests := []struct { + name string + pvc *v1.PersistentVolumeClaim + featureGateEnabled bool + expectedRecoveryCheck bool + }{ + { + name: "feature gate disabled, no resize status or allocated resources", + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc-1", + }, + Status: v1.PersistentVolumeClaimStatus{ + AllocatedResourceStatuses: nil, + AllocatedResources: nil, + }, + }, + featureGateEnabled: false, + expectedRecoveryCheck: false, + }, + { + name: "feature gate disabled, resize status set", + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc-2", + }, + Status: v1.PersistentVolumeClaimStatus{ + AllocatedResourceStatuses: map[v1.ResourceName]v1.ClaimResourceStatus{ + v1.ResourceStorage: v1.PersistentVolumeClaimNodeResizePending, + }, + }, + }, + featureGateEnabled: false, + expectedRecoveryCheck: true, + }, + { + name: "feature gate enabled, resize status and allocated resources set", + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc-3", + }, + Status: v1.PersistentVolumeClaimStatus{ + AllocatedResourceStatuses: map[v1.ResourceName]v1.ClaimResourceStatus{ + v1.ResourceStorage: v1.PersistentVolumeClaimNodeResizePending, + }, + AllocatedResources: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("10Gi"), + }, + }, + }, + featureGateEnabled: true, + expectedRecoveryCheck: true, + }, + { + name: "feature gate enabled, no resize status or allocated resources", + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc-4", + }, + Status: v1.PersistentVolumeClaimStatus{ + AllocatedResourceStatuses: nil, + AllocatedResources: nil, + }, + }, + featureGateEnabled: true, + expectedRecoveryCheck: false, + }, + { + name: "feature gate enabled, older external resize controller", + pvc: &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc-5", + }, + Status: v1.PersistentVolumeClaimStatus{ + AllocatedResourceStatuses: nil, + AllocatedResources: nil, + }, + }, + featureGateEnabled: true, + expectedRecoveryCheck: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, test.featureGateEnabled) + + pod := getTestPod("test-pod", test.pvc.Name) + pv := getTestPV("test-vol0", "2G") + og := &operationGenerator{} + + vmt := VolumeToMount{ + Pod: pod, + VolumeName: v1.UniqueVolumeName(pv.Name), + VolumeSpec: volume.NewSpecFromPersistentVolume(pv, false), + } + result := og.checkForRecoveryFromExpansion(test.pvc, vmt) + + assert.Equal(t, test.expectedRecoveryCheck, result, "unexpected recovery check result for test: %s", test.name) + }) + } +} func getTestPod(podName, pvcName string) *v1.Pod { return &v1.Pod{ From 69e0af31000bafbf27111ac3a483cfb950e9dadf Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Wed, 23 Apr 2025 16:04:58 -0400 Subject: [PATCH 04/11] Also change final status by removing featuregate check --- .../util/operationexecutor/node_expander.go | 2 +- pkg/volume/util/resize_util.go | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/volume/util/operationexecutor/node_expander.go b/pkg/volume/util/operationexecutor/node_expander.go index fff1760d0f5ef..fb4366df0601b 100644 --- a/pkg/volume/util/operationexecutor/node_expander.go +++ b/pkg/volume/util/operationexecutor/node_expander.go @@ -184,7 +184,7 @@ func (ne *NodeExpander) expandOnPlugin() (bool, resource.Quantity, error) { } // File system resize succeeded, now update the PVC's Capacity to match the PV's - ne.pvc, err = util.MarkFSResizeFinished(ne.pvc, ne.pluginResizeOpts.NewSize, ne.kubeClient) + ne.pvc, err = util.MarkNodeExpansionFinishedWithRecovery(ne.pvc, ne.pluginResizeOpts.NewSize, ne.kubeClient) if err != nil { return true, ne.pluginResizeOpts.NewSize, fmt.Errorf("mountVolume.NodeExpandVolume update pvc status failed: %w", err) } diff --git a/pkg/volume/util/resize_util.go b/pkg/volume/util/resize_util.go index 2bf54b4b85cc9..80b1845308f13 100644 --- a/pkg/volume/util/resize_util.go +++ b/pkg/volume/util/resize_util.go @@ -236,6 +236,28 @@ func MarkFSResizeFinished( return updatedPVC, err } +func MarkNodeExpansionFinishedWithRecovery( + pvc *v1.PersistentVolumeClaim, + newSize resource.Quantity, + kubeClient clientset.Interface) (*v1.PersistentVolumeClaim, error) { + newPVC := pvc.DeepCopy() + + newPVC.Status.Capacity[v1.ResourceStorage] = newSize + + // if RecoverVolumeExpansionFailure is enabled, we need to reset ResizeStatus back to nil + allocatedResourceStatusMap := newPVC.Status.AllocatedResourceStatuses + delete(allocatedResourceStatusMap, v1.ResourceStorage) + if len(allocatedResourceStatusMap) == 0 { + newPVC.Status.AllocatedResourceStatuses = nil + } else { + newPVC.Status.AllocatedResourceStatuses = allocatedResourceStatusMap + } + + newPVC = MergeResizeConditionOnPVC(newPVC, []v1.PersistentVolumeClaimCondition{}, false /* keepOldResizeConditions */) + updatedPVC, err := PatchPVCStatus(pvc /*oldPVC*/, newPVC, kubeClient) + return updatedPVC, err +} + // MarkNodeExpansionInfeasible marks a PVC for node expansion as failed. Kubelet should not retry expansion // of volumes which are in failed state. func MarkNodeExpansionInfeasible(pvc *v1.PersistentVolumeClaim, kubeClient clientset.Interface, err error) (*v1.PersistentVolumeClaim, error) { From cdbea02d39fe4f13b52fd316f37d14c34d0a7798 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Wed, 23 Apr 2025 16:10:49 -0400 Subject: [PATCH 05/11] Mark NodeExpansion finsihed without featuregate check --- .../operationexecutor/node_expander_test.go | 88 ++++++++++++------- pkg/volume/util/resize_util.go | 1 - 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/pkg/volume/util/operationexecutor/node_expander_test.go b/pkg/volume/util/operationexecutor/node_expander_test.go index e95a42d5ce33d..01c3f75253f93 100644 --- a/pkg/volume/util/operationexecutor/node_expander_test.go +++ b/pkg/volume/util/operationexecutor/node_expander_test.go @@ -49,9 +49,10 @@ func TestNodeExpander(t *testing.T) { nodeResizePending := v1.PersistentVolumeClaimNodeResizePending var tests = []struct { - name string - pvc *v1.PersistentVolumeClaim - pv *v1.PersistentVolume + name string + pvc *v1.PersistentVolumeClaim + pv *v1.PersistentVolume + recoverVolumeExpansionFailure bool // desired size, defaults to pv.Spec.Capacity desiredSize *resource.Quantity @@ -67,9 +68,10 @@ func TestNodeExpander(t *testing.T) { expectError bool }{ { - name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_failed", - pvc: getTestPVC("test-vol0", "2G", "1G", "", &nodeResizeFailed), - pv: getTestPV("test-vol0", "2G"), + name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_failed", + pvc: getTestPVC("test-vol0", "2G", "1G", "", &nodeResizeFailed), + pv: getTestPV("test-vol0", "2G"), + recoverVolumeExpansionFailure: true, expectedResizeStatus: nodeResizeFailed, expectResizeCall: false, @@ -78,9 +80,11 @@ func TestNodeExpander(t *testing.T) { expectedStatusSize: resource.MustParse("1G"), }, { - name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending", - pvc: getTestPVC("test-vol0", "2G", "1G", "2G", &nodeResizePending), - pv: getTestPV("test-vol0", "2G"), + name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending", + pvc: getTestPVC("test-vol0", "2G", "1G", "2G", &nodeResizePending), + pv: getTestPV("test-vol0", "2G"), + recoverVolumeExpansionFailure: true, + expectedResizeStatus: "", expectResizeCall: true, assumeResizeOpAsFinished: true, @@ -88,31 +92,34 @@ func TestNodeExpander(t *testing.T) { expectedStatusSize: resource.MustParse("2G"), }, { - name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending, reize_op=infeasible", - pvc: getTestPVC(volumetesting.InfeasibleNodeExpansion, "2G", "1G", "2G", &nodeResizePending), - pv: getTestPV(volumetesting.InfeasibleNodeExpansion, "2G"), - expectError: true, - expectedResizeStatus: nodeResizeFailed, - expectResizeCall: true, - assumeResizeOpAsFinished: true, - expectFinalErrors: true, - expectedStatusSize: resource.MustParse("1G"), + name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending, reize_op=infeasible", + pvc: getTestPVC(volumetesting.InfeasibleNodeExpansion, "2G", "1G", "2G", &nodeResizePending), + pv: getTestPV(volumetesting.InfeasibleNodeExpansion, "2G"), + recoverVolumeExpansionFailure: false, + expectError: true, + expectedResizeStatus: nodeResizeFailed, + expectResizeCall: true, + assumeResizeOpAsFinished: true, + expectFinalErrors: true, + expectedStatusSize: resource.MustParse("1G"), }, { - name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending, reize_op=failing", - pvc: getTestPVC(volumetesting.OtherFinalNodeExpansionError, "2G", "1G", "2G", &nodeResizePending), - pv: getTestPV(volumetesting.OtherFinalNodeExpansionError, "2G"), - expectError: true, - expectedResizeStatus: v1.PersistentVolumeClaimNodeResizeInProgress, - expectResizeCall: true, - assumeResizeOpAsFinished: true, - expectFinalErrors: true, - expectedStatusSize: resource.MustParse("1G"), + name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending, reize_op=failing", + pvc: getTestPVC(volumetesting.OtherFinalNodeExpansionError, "2G", "1G", "2G", &nodeResizePending), + pv: getTestPV(volumetesting.OtherFinalNodeExpansionError, "2G"), + recoverVolumeExpansionFailure: true, + expectError: true, + expectedResizeStatus: v1.PersistentVolumeClaimNodeResizeInProgress, + expectResizeCall: true, + assumeResizeOpAsFinished: true, + expectFinalErrors: true, + expectedStatusSize: resource.MustParse("1G"), }, { - name: "RWO volumes, pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize", - pvc: getTestPVC("test-vol0", "2G", "2G", "2G", nil), - pv: getTestPV("test-vol0", "2G"), + name: "RWO volumes, pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize", + pvc: getTestPVC("test-vol0", "2G", "2G", "2G", nil), + pv: getTestPV("test-vol0", "2G"), + recoverVolumeExpansionFailure: false, expectedResizeStatus: "", expectResizeCall: false, @@ -121,9 +128,22 @@ func TestNodeExpander(t *testing.T) { expectedStatusSize: resource.MustParse("2G"), }, { - name: "RWX volumes, pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize", - pvc: addAccessMode(getTestPVC("test-vol0", "2G", "2G", "2G", nil), v1.ReadWriteMany), - pv: getTestPV("test-vol0", "2G"), + name: "RWX volumes, pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize", + pvc: addAccessMode(getTestPVC("test-vol0", "2G", "2G", "2G", nil), v1.ReadWriteMany), + pv: getTestPV("test-vol0", "2G"), + recoverVolumeExpansionFailure: true, + + expectedResizeStatus: "", + expectResizeCall: true, + assumeResizeOpAsFinished: true, + expectFinalErrors: false, + expectedStatusSize: resource.MustParse("2G"), + }, + { + name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending, featuregate=disabled", + pvc: getTestPVC("test-vol0", "2G", "1G", "2G", &nodeResizePending), + pv: getTestPV("test-vol0", "2G"), + recoverVolumeExpansionFailure: false, expectedResizeStatus: "", expectResizeCall: true, @@ -136,7 +156,7 @@ func TestNodeExpander(t *testing.T) { for i := range tests { test := tests[i] t.Run(test.name, func(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, true) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, test.recoverVolumeExpansionFailure) volumePluginMgr, fakePlugin := volumetesting.GetTestKubeletVolumePluginMgr(t) pvc := test.pvc diff --git a/pkg/volume/util/resize_util.go b/pkg/volume/util/resize_util.go index 80b1845308f13..599f220976e7a 100644 --- a/pkg/volume/util/resize_util.go +++ b/pkg/volume/util/resize_util.go @@ -244,7 +244,6 @@ func MarkNodeExpansionFinishedWithRecovery( newPVC.Status.Capacity[v1.ResourceStorage] = newSize - // if RecoverVolumeExpansionFailure is enabled, we need to reset ResizeStatus back to nil allocatedResourceStatusMap := newPVC.Status.AllocatedResourceStatuses delete(allocatedResourceStatusMap, v1.ResourceStorage) if len(allocatedResourceStatusMap) == 0 { From b23e637969f09dd2dafb956b466752c9057ad3f1 Mon Sep 17 00:00:00 2001 From: Prince Pereira Date: Sun, 27 Apr 2025 21:52:32 -0700 Subject: [PATCH 06/11] Manually adding fix for failing pull-kubernetes-typecheck tests which had dependency with this commit: https://github.com/kubernetes/kubernetes/commit/61863035803cc105619925a0fa23a779a9578bd4 --- pkg/proxy/winkernel/proxier_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/proxy/winkernel/proxier_test.go b/pkg/proxy/winkernel/proxier_test.go index 293cce56dc43d..6064f97e52683 100644 --- a/pkg/proxy/winkernel/proxier_test.go +++ b/pkg/proxy/winkernel/proxier_test.go @@ -129,6 +129,26 @@ func NewFakeProxier(syncPeriod time.Duration, minSyncPeriod time.Duration, hostn return proxier } +func getHcnMock(networkType string) *fakehcn.HcnMock { + var remoteSubnets []*remoteSubnetInfo + rs := &remoteSubnetInfo{ + destinationPrefix: destinationPrefix, + isolationID: 4096, + providerAddress: providerAddress, + drMacAddress: macAddress, + } + remoteSubnets = append(remoteSubnets, rs) + hnsNetworkInfo := &hnsNetworkInfo{ + id: strings.ToUpper(guid), + name: testNetwork, + networkType: networkType, + remoteSubnets: remoteSubnets, + } + hnsNetwork := newHnsNetwork(hnsNetworkInfo) + hcnMock := fakehcn.NewHcnMock(hnsNetwork) + return hcnMock +} + func TestCreateServiceVip(t *testing.T) { syncPeriod := 30 * time.Second proxier := NewFakeProxier(syncPeriod, syncPeriod, "testhost", netutils.ParseIPSloppy("10.0.0.1"), NETWORK_TYPE_OVERLAY) From 5ad726c6ef3301e9816410347e5a7764332375ef Mon Sep 17 00:00:00 2001 From: carlory Date: Sun, 27 Apr 2025 11:21:36 +0800 Subject: [PATCH 07/11] Handle unsupported node expansion for RWX volumes Co-authored-by: Hemant Kumar Signed-off-by: carlory --- .../util/operationexecutor/node_expander.go | 20 +++++++++++++++++++ .../operationexecutor/node_expander_test.go | 11 ++++++++++ 2 files changed, 31 insertions(+) diff --git a/pkg/volume/util/operationexecutor/node_expander.go b/pkg/volume/util/operationexecutor/node_expander.go index fff1760d0f5ef..25ae208f8c4f3 100644 --- a/pkg/volume/util/operationexecutor/node_expander.go +++ b/pkg/volume/util/operationexecutor/node_expander.go @@ -144,6 +144,26 @@ func (ne *NodeExpander) expandOnPlugin() (bool, resource.Quantity, error) { } _, resizeErr := ne.volumePlugin.NodeExpand(ne.pluginResizeOpts) if resizeErr != nil { + // In order to support node volume expansion for RWX volumes on different nodes, + // we bypass the check for VolumeExpansionPendingOnNode state during the pre-check + // and then directly call the NodeExpandVolume method on the plugin. + // + // However, it does not make sense where the csi driver does not support node expansion. + // We should not treat this as a failure. It is a workaround for this issue: + // https://github.com/kubernetes/kubernetes/issues/131381. + // + // For other access modes, we should not hit this state, because we will wait for + // VolumeExpansionPendingOnNode before trying to expand volume in kubelet. + // See runPreCheck() above. + // + // If volume is already expanded, then we should not retry expansion on the node if + // driver returns OperationNotSupportedError. + if volumetypes.IsOperationNotSupportedError(resizeErr) && ne.pvcAlreadyUpdated { + klog.V(4).InfoS(ne.vmt.GenerateMsgDetailed("MountVolume.NodeExpandVolume failed", "NodeExpandVolume not supported"), "pod", klog.KObj(ne.vmt.Pod)) + ne.testStatus = testResponseData{assumeResizeFinished: true, resizeCalledOnPlugin: false} + return true, ne.pluginResizeOpts.NewSize, nil + } + if volumetypes.IsOperationFinishedError(resizeErr) { var markFailedError error ne.actualStateOfWorld.MarkVolumeExpansionFailedWithFinalError(ne.vmt.VolumeName) diff --git a/pkg/volume/util/operationexecutor/node_expander_test.go b/pkg/volume/util/operationexecutor/node_expander_test.go index e95a42d5ce33d..a4c3ade0ea1df 100644 --- a/pkg/volume/util/operationexecutor/node_expander_test.go +++ b/pkg/volume/util/operationexecutor/node_expander_test.go @@ -131,6 +131,17 @@ func TestNodeExpander(t *testing.T) { expectFinalErrors: false, expectedStatusSize: resource.MustParse("2G"), }, + { + name: "RWX pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize, reize_op=unsupported", + pvc: addAccessMode(getTestPVC(volumetesting.FailWithUnSupportedVolumeName, "2G", "2G", "2G", nil), v1.ReadWriteMany), + pv: getTestPV(volumetesting.FailWithUnSupportedVolumeName, "2G"), + expectError: false, + expectedResizeStatus: "", + expectResizeCall: false, + assumeResizeOpAsFinished: true, + expectFinalErrors: false, + expectedStatusSize: resource.MustParse("2G"), + }, } for i := range tests { From 7ba97788a71b0c04c6b3831480ece4e6dc4c94e6 Mon Sep 17 00:00:00 2001 From: Arnaud Meukam Date: Tue, 29 Apr 2025 11:56:47 +0200 Subject: [PATCH 08/11] Bump images, dependencies and versions Bump images, dependencies, versions to Go 1.23.8 Bump distroless-iptables to 0.6.9 Signed-off-by: Arnaud Meukam --- .go-version | 2 +- build/build-image/cross/VERSION | 2 +- build/common.sh | 4 ++-- build/dependencies.yaml | 8 ++++---- staging/publishing/rules.yaml | 2 +- test/images/Makefile | 2 +- test/utils/image/manifest.go | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.go-version b/.go-version index fa994bd719710..82bfa5ce3fc25 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.23.6 \ No newline at end of file +1.23.8 diff --git a/build/build-image/cross/VERSION b/build/build-image/cross/VERSION index 7aeec2e2274b7..f1b969720e2bb 100644 --- a/build/build-image/cross/VERSION +++ b/build/build-image/cross/VERSION @@ -1 +1 @@ -v1.32.0-go1.23.6-bullseye.0 +v1.32.0-go1.23.8-bullseye.0 diff --git a/build/common.sh b/build/common.sh index ef44ce99f27ed..6e1aa07b99489 100755 --- a/build/common.sh +++ b/build/common.sh @@ -97,8 +97,8 @@ readonly KUBE_RSYNC_PORT="${KUBE_RSYNC_PORT:-}" readonly KUBE_CONTAINER_RSYNC_PORT=8730 # These are the default versions (image tags) for their respective base images. -readonly __default_distroless_iptables_version=v0.6.8 -readonly __default_go_runner_version=v2.4.0-go1.23.6-bookworm.0 +readonly __default_distroless_iptables_version=v0.6.9 +readonly __default_go_runner_version=v2.4.0-go1.23.8-bookworm.0 readonly __default_setcap_version=bookworm-v1.0.4 # These are the base images for the Docker-wrapped binaries. diff --git a/build/dependencies.yaml b/build/dependencies.yaml index 3b60db44a1e91..5bfae6a58efd2 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -116,7 +116,7 @@ dependencies: # Golang - name: "golang: upstream version" - version: 1.23.6 + version: 1.23.8 refPaths: - path: .go-version - path: build/build-image/cross/VERSION @@ -140,7 +140,7 @@ dependencies: match: golang:([0-9]+\.[0-9]+).0-bullseye - name: "registry.k8s.io/kube-cross: dependents" - version: v1.32.0-go1.23.6-bullseye.0 + version: v1.32.0-go1.23.8-bullseye.0 refPaths: - path: build/build-image/cross/VERSION @@ -178,7 +178,7 @@ dependencies: match: registry\.k8s\.io\/build-image\/debian-base:[a-zA-Z]+\-v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?) - name: "registry.k8s.io/distroless-iptables: dependents" - version: v0.6.8 + version: v0.6.9 refPaths: - path: build/common.sh match: __default_distroless_iptables_version= @@ -186,7 +186,7 @@ dependencies: match: configs\[DistrolessIptables\] = Config{list\.BuildImageRegistry, "distroless-iptables", "v([0-9]+)\.([0-9]+)\.([0-9]+)"} - name: "registry.k8s.io/go-runner: dependents" - version: v2.4.0-go1.23.6-bookworm.0 + version: v2.4.0-go1.23.8-bookworm.0 refPaths: - path: build/common.sh match: __default_go_runner_version= diff --git a/staging/publishing/rules.yaml b/staging/publishing/rules.yaml index 526db0a4640bb..d07783f16b107 100644 --- a/staging/publishing/rules.yaml +++ b/staging/publishing/rules.yaml @@ -2900,4 +2900,4 @@ rules: - staging/src/k8s.io/externaljwt recursive-delete-patterns: - '*/.gitattributes' -default-go-version: 1.23.6 +default-go-version: 1.23.8 diff --git a/test/images/Makefile b/test/images/Makefile index a2bd6fb83a3b1..65910b90e4af8 100644 --- a/test/images/Makefile +++ b/test/images/Makefile @@ -16,7 +16,7 @@ REGISTRY ?= registry.k8s.io/e2e-test-images GOARM ?= 7 DOCKER_CERT_BASE_PATH ?= QEMUVERSION=v5.1.0-2 -GOLANG_VERSION=1.23.6 +GOLANG_VERSION=1.23.8 export ifndef WHAT diff --git a/test/utils/image/manifest.go b/test/utils/image/manifest.go index 02876dd944d79..3210068bf3050 100644 --- a/test/utils/image/manifest.go +++ b/test/utils/image/manifest.go @@ -223,7 +223,7 @@ func initImageConfigs(list RegistryList) (map[ImageID]Config, map[ImageID]Config configs[APIServer] = Config{list.PromoterE2eRegistry, "sample-apiserver", "1.29.2"} configs[AppArmorLoader] = Config{list.PromoterE2eRegistry, "apparmor-loader", "1.4"} configs[BusyBox] = Config{list.PromoterE2eRegistry, "busybox", "1.36.1-1"} - configs[DistrolessIptables] = Config{list.BuildImageRegistry, "distroless-iptables", "v0.6.8"} + configs[DistrolessIptables] = Config{list.BuildImageRegistry, "distroless-iptables", "v0.6.9"} configs[Etcd] = Config{list.GcEtcdRegistry, "etcd", "3.5.16-0"} configs[Httpd] = Config{list.PromoterE2eRegistry, "httpd", "2.4.38-4"} configs[HttpdNew] = Config{list.PromoterE2eRegistry, "httpd", "2.4.39-4"} From 9894294ef13a5b32803e3ca2c0d620a088cc84d1 Mon Sep 17 00:00:00 2001 From: Kubernetes Release Robot Date: Thu, 15 May 2025 09:05:02 +0000 Subject: [PATCH 09/11] Release commit for Kubernetes v1.32.5 From 8ed7d1c099caadd9db73e57e05efc9509f78aa04 Mon Sep 17 00:00:00 2001 From: Allen Ray Date: Wed, 21 May 2025 11:16:21 -0400 Subject: [PATCH 10/11] UPSTREAM: : manually resolve conflicts --- .../operationexecutor/node_expander_test.go | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/pkg/volume/util/operationexecutor/node_expander_test.go b/pkg/volume/util/operationexecutor/node_expander_test.go index 2007eba8a197b..6b339d71c9b13 100644 --- a/pkg/volume/util/operationexecutor/node_expander_test.go +++ b/pkg/volume/util/operationexecutor/node_expander_test.go @@ -128,7 +128,6 @@ func TestNodeExpander(t *testing.T) { expectedStatusSize: resource.MustParse("2G"), }, { -<<<<<<< HEAD name: "RWX volumes, pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize", pvc: addAccessMode(getTestPVC("test-vol0", "2G", "2G", "2G", nil), v1.ReadWriteMany), pv: getTestPV("test-vol0", "2G"), @@ -140,40 +139,6 @@ func TestNodeExpander(t *testing.T) { expectFinalErrors: false, expectedStatusSize: resource.MustParse("2G"), }, - { - name: "RWX pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize, reize_op=unsupported", - pvc: addAccessMode(getTestPVC(volumetesting.FailWithUnSupportedVolumeName, "2G", "2G", "2G", nil), v1.ReadWriteMany), - pv: getTestPV(volumetesting.FailWithUnSupportedVolumeName, "2G"), - recoverVolumeExpansionFailure: true, - expectError: false, - expectedResizeStatus: "", - expectResizeCall: false, - assumeResizeOpAsFinished: true, - expectFinalErrors: false, - expectedStatusSize: resource.MustParse("2G"), - }, - { - name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending, featuregate=disabled", - pvc: getTestPVC("test-vol0", "2G", "1G", "2G", &nodeResizePending), - pv: getTestPV("test-vol0", "2G"), - recoverVolumeExpansionFailure: false, -||||||| 59526cd4867 - name: "RWX volumes, pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize", - pvc: addAccessMode(getTestPVC("test-vol0", "2G", "2G", "2G", nil), v1.ReadWriteMany), - pv: getTestPV("test-vol0", "2G"), -======= - name: "RWX volumes, pv.spec.cap = pvc.status.cap, resizeStatus='', desiredSize > actualSize", - pvc: addAccessMode(getTestPVC("test-vol0", "2G", "2G", "2G", nil), v1.ReadWriteMany), - pv: getTestPV("test-vol0", "2G"), - recoverVolumeExpansionFailure: true, ->>>>>>> v1.32.5 - - expectedResizeStatus: "", - expectResizeCall: true, - assumeResizeOpAsFinished: true, - expectFinalErrors: false, - expectedStatusSize: resource.MustParse("2G"), - }, { name: "pv.spec.cap > pvc.status.cap, resizeStatus=node_expansion_pending, featuregate=disabled", pvc: getTestPVC("test-vol0", "2G", "1G", "2G", &nodeResizePending), From 01fefa3690448dfbf3734914eaaf11e8ac310b26 Mon Sep 17 00:00:00 2001 From: Allen Ray Date: Wed, 21 May 2025 11:35:15 -0400 Subject: [PATCH 11/11] UPSTREAM: : hack/update-vendor.sh, make update and update image --- openshift-hack/images/hyperkube/Dockerfile.rhel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openshift-hack/images/hyperkube/Dockerfile.rhel b/openshift-hack/images/hyperkube/Dockerfile.rhel index 757c2f958f5e7..732dfea12ff03 100644 --- a/openshift-hack/images/hyperkube/Dockerfile.rhel +++ b/openshift-hack/images/hyperkube/Dockerfile.rhel @@ -14,4 +14,4 @@ COPY --from=builder /tmp/build/* /usr/bin/ LABEL io.k8s.display-name="OpenShift Kubernetes Server Commands" \ io.k8s.description="OpenShift is a platform for developing, building, and deploying containerized applications." \ io.openshift.tags="openshift,hyperkube" \ - io.openshift.build.versions="kubernetes=1.32.4" \ No newline at end of file + io.openshift.build.versions="kubernetes=1.32.5" \ No newline at end of file