This is an automated email from the ASF dual-hosted git repository. Pearl1594 pushed a commit to branch cks-offline-upgrade in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 220e4b2e59bba7d0e6ca6fb77cd52efb8ebf372e Author: Pearl Dsilva <[email protected]> AuthorDate: Tue Jun 16 22:34:50 2026 -0400 Pre-seed images on worker nodes before control plane upgrade for air-gapped envs --- .../KubernetesClusterUpgradeWorker.java | 45 ++++++++++++++++++++++ .../main/resources/script/upgrade-kubernetes.sh | 10 +++++ 2 files changed, 55 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 4c2725fc2a2..9f59b04c44d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -63,6 +63,20 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke upgradeScriptFile = retrieveScriptFile(upgradeScriptFilename); } + private Pair<Boolean, String> preseedImagesOnVM(final UserVm vm, final int index) throws Exception { + int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index; + String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; + SshHelper.scpTo(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, + "~/", upgradeScriptFile.getAbsolutePath(), "0755"); + String cmdStr = String.format("sudo ./%s %s false false %s %s true", + upgradeScriptFile.getName(), + upgradeVersion.getSemanticVersion(), + Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType()), + Objects.isNull(kubernetesCluster.getCniConfigId())); + return SshHelper.sshExecute(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, + cmdStr, 10000, 10000, 5 * 60 * 1000); + } + private Pair<Boolean, String> runInstallScriptOnVM(final UserVm vm, final int index) throws Exception { int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index; String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; @@ -80,6 +94,37 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke } private void upgradeKubernetesClusterNodes() { + // kubeadm upgrade apply schedules a post-upgrade health check pod on a non-cordoned node. + // In air-gapped clusters, workers lack the new pause image until their own upgrade runs, + // causing the pod pull to fail and the upgrade to stall. Pre-seed all workers first so + // the image is present before the control-plane upgrade starts. + if (clusterVMs.size() > 1) { + for (int i = 1; i < clusterVMs.size(); ++i) { + UserVm vm = clusterVMs.get(i); + String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to pre-seed images on VM : %s", + kubernetesCluster.getName(), vm.getDisplayName()); + for (int retry = KubernetesClusterService.KubernetesClusterUpgradeRetries.value(); retry >= 0; retry--) { + try { + Pair<Boolean, String> result = preseedImagesOnVM(vm, i); + if (result.first()) { + break; + } + if (retry > 0) { + logger.warn("{}, retries left: {}", errorMessage, retry); + } else { + logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } + } catch (Exception e) { + if (retry > 0) { + logger.warn("{} due to {}, retries left: {}", errorMessage, e, retry); + } else { + logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); + } + } + } + } + } + for (int i = 0; i < clusterVMs.size(); ++i) { UserVm vm = clusterVMs.get(i); String hostName = vm.getHostName(); diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index a947d508436..494811a00de 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -39,6 +39,10 @@ EXTERNAL_CNI=false if [ $# -gt 4 ]; then EXTERNAL_CNI="${5}" fi +PRESEED_ONLY=false +if [ $# -gt 5 ]; then + PRESEED_ONLY="${6}" +fi export PATH=$PATH:/opt/bin if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then @@ -120,6 +124,12 @@ if [ -d "$BINARIES_DIR" ]; then sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml fi + if [ "${PRESEED_ONLY}" == 'true' ]; then + systemctl restart containerd + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + exit 0 + fi + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz
