This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch add-support-csi-projects
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/add-support-csi-projects by
this push:
new cbdce2d4bfe Add support to setup csi driver on k8s cluster creation
cbdce2d4bfe is described below
commit cbdce2d4bfed47de3c0bcce51c0900d3fed26bc5
Author: Pearl Dsilva <pearl1...@gmail.com>
AuthorDate: Mon Jul 28 14:48:19 2025 -0400
Add support to setup csi driver on k8s cluster creation
---
.../kubernetes/cluster/KubernetesCluster.java | 1 +
.../org/apache/cloudstack/api/ApiConstants.java | 1 +
.../resources/META-INF/db/schema-42010to42100.sql | 3 ++
.../cluster/KubernetesClusterManagerImpl.java | 1 +
.../kubernetes/cluster/KubernetesClusterVO.java | 11 +++++
.../KubernetesClusterActionWorker.java | 41 ++++++++++++++++++
.../KubernetesClusterStartWorker.java | 9 +++-
.../cluster/CreateKubernetesClusterCmd.java | 7 +++
.../src/main/resources/conf/k8s-control-node.yml | 5 +++
.../src/main/resources/script/deploy-csi-driver | 50 ++++++++++++++++++++++
scripts/util/create-kubernetes-binaries-iso.sh | 7 +++
ui/public/locales/en.json | 1 +
ui/src/views/compute/CreateKubernetesCluster.vue | 10 +++++
13 files changed, 145 insertions(+), 2 deletions(-)
diff --git
a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java
b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java
index 8b5551d6a8e..83457a2aef5 100644
--- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java
+++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java
@@ -166,4 +166,5 @@ public interface KubernetesCluster extends
ControlledEntity, com.cloud.utils.fsm
Long getEtcdNodeCount();
Long getCniConfigId();
String getCniConfigDetails();
+ boolean isCsiEnabled();
}
diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
index 22e7e807502..e3926a56d78 100644
--- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
@@ -196,6 +196,7 @@ public class ApiConstants {
public static final String DURATION = "duration";
public static final String ELIGIBLE = "eligible";
public static final String EMAIL = "email";
+ public static final String ENABLE_CSI = "enablecsi";
public static final String END_ASN = "endasn";
public static final String END_DATE = "enddate";
public static final String END_IP = "endip";
diff --git
a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql
b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql
index 5a50b96d8f2..4b2b86043c1 100644
--- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql
+++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql
@@ -203,3 +203,6 @@ SET `sort_key` = CASE
ELSE `sort_key`
END;
-- End: Changes for Guest OS category cleanup
+
+-- Add csi_enabled column to kubernetes_cluster table - Move to 4.22
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes', 'csi_enabled',
'TINYINT(1) unsigned NOT NULL DEFAULT 0 COMMENT "true if kubernetes cluster is
using csi, false otherwise" ');
\ No newline at end of file
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
index b922d80727d..a39b5bba58c 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
@@ -1604,6 +1604,7 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
if (zone.isSecurityGroupEnabled()) {
newCluster.setSecurityGroupId(finalSecurityGroup.getId());
}
+ newCluster.setCsiEnabled(cmd.getEnableCsi());
kubernetesClusterDao.persist(newCluster);
addKubernetesClusterDetails(newCluster, defaultNetwork, cmd);
return newCluster;
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java
index 79fc15f6898..7dfd0043e32 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java
@@ -145,6 +145,9 @@ public class KubernetesClusterVO implements
KubernetesCluster {
@Column(name = "cni_config_details", updatable = true, length = 4096)
private String cniConfigDetails;
+ @Column(name = "csi_enabled")
+ private boolean csiEnabled;
+
@Override
public long getId() {
return id;
@@ -389,6 +392,14 @@ public class KubernetesClusterVO implements
KubernetesCluster {
this.clusterType = clusterType;
}
+ public boolean isCsiEnabled() {
+ return csiEnabled;
+ }
+
+ public void setCsiEnabled(boolean csiEnabled) {
+ this.csiEnabled = csiEnabled;
+ }
+
public KubernetesClusterVO() {
this.uuid = UUID.randomUUID().toString();
}
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
index 7ce227dfeb7..843c5506e9f 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
@@ -231,12 +231,14 @@ public class KubernetesClusterActionWorker {
protected final String deploySecretsScriptFilename =
"deploy-cloudstack-secret";
protected final String deployProviderScriptFilename = "deploy-provider";
+ protected final String deployCsiDriverScriptFilename = "deploy-csi-driver";
protected final String autoscaleScriptFilename = "autoscale-kube-cluster";
protected final String validateNodeScript = "validate-cks-node";
protected final String removeNodeFromClusterScript =
"remove-node-from-cluster";
protected final String scriptPath = "/opt/bin/";
protected File deploySecretsScriptFile;
protected File deployProviderScriptFile;
+ protected File deployCsiDriverScriptFile;
protected File autoscaleScriptFile;
protected KubernetesClusterManagerImpl manager;
protected String[] keys;
@@ -713,12 +715,14 @@ public class KubernetesClusterActionWorker {
protected void retrieveScriptFiles() {
deploySecretsScriptFile =
retrieveScriptFile(deploySecretsScriptFilename);
deployProviderScriptFile =
retrieveScriptFile(deployProviderScriptFilename);
+ deployCsiDriverScriptFile =
retrieveScriptFile(deployCsiDriverScriptFilename);
autoscaleScriptFile = retrieveScriptFile(autoscaleScriptFilename);
}
protected void copyScripts(String nodeAddress, final int sshPort) {
copyScriptFile(nodeAddress, sshPort, deploySecretsScriptFile,
deploySecretsScriptFilename);
copyScriptFile(nodeAddress, sshPort, deployProviderScriptFile,
deployProviderScriptFilename);
+ copyScriptFile(nodeAddress, sshPort, deployCsiDriverScriptFile,
deployCsiDriverScriptFilename);
copyScriptFile(nodeAddress, sshPort, autoscaleScriptFile,
autoscaleScriptFilename);
}
@@ -819,6 +823,43 @@ public class KubernetesClusterActionWorker {
}
}
+ protected boolean deployCsiDriver() {
+ File pkFile = getManagementServerSshPublicKeyFile();
+ Pair<String, Integer> publicIpSshPort =
getKubernetesClusterServerIpSshPort(null);
+ publicIpAddress = publicIpSshPort.first();
+ sshPort = publicIpSshPort.second();
+
+ try {
+ String command = String.format("sudo %s/%s", scriptPath,
deployCsiDriverScriptFilename);
+ Pair<Boolean, String> result =
SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
+ pkFile, null, command, 10000, 10000, 60000);
+
+ // Maybe the file isn't present. Try and copy it
+ if (!result.first()) {
+ logMessage(Level.INFO, "CSI files missing. Adding them now",
null);
+ retrieveScriptFiles();
+ copyScripts(publicIpAddress, sshPort);
+
+ if (!createCloudStackSecret(keys)) {
+ logTransitStateAndThrow(Level.ERROR, String.format("Failed
to setup keys for Kubernetes cluster %s",
+ kubernetesCluster.getName()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
+ }
+
+ // If at first you don't succeed ...
+ result = SshHelper.sshExecute(publicIpAddress, sshPort,
getControlNodeLoginUser(),
+ pkFile, null, command, 10000, 10000, 60000);
+ if (!result.first()) {
+ throw new CloudRuntimeException(result.second());
+ }
+ }
+ return true;
+ } catch (Exception e) {
+ String msg = String.format("Failed to deploy kubernetes provider:
%s : %s", kubernetesCluster.getName(), e.getMessage());
+ logAndThrow(Level.ERROR, msg);
+ return false;
+ }
+ }
+
public void setKeys(String[] keys) {
this.keys = keys;
}
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
index 68bec58d462..227bb26a24c 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
@@ -143,7 +143,7 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
private Pair<String, String> getKubernetesControlNodeConfig(final String
controlNodeIp, final String serverIp,
final
List<Network.IpAddresses> etcdIps, final String hostName, final boolean
haSupported,
- final boolean
ejectIso, final boolean externalCni) throws IOException {
+ final boolean
ejectIso, final boolean externalCni, final boolean setupCsi) throws IOException
{
String k8sControlNodeConfig =
readK8sConfigFile("/conf/k8s-control-node.yml");
final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}";
final String apiServerKey = "{{ k8s_control_node.apiserver.key }}";
@@ -161,6 +161,7 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
final String certSans = "{{ k8s_control.server_ips }}";
final String k8sCertificate = "{{ k8s_control.certificate_key }}";
final String externalCniPlugin = "{{ k8s.external.cni.plugin }}";
+ final String setupCsiDriver = "{{ k8s.setup.csi.driver }}";
final List<String> addresses = new ArrayList<>();
addresses.add(controlNodeIp);
@@ -212,6 +213,7 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans,
String.format("- %s", serverIp));
k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate,
KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin,
String.valueOf(externalCni));
+ k8sControlNodeConfig = k8sControlNodeConfig.replace(setupCsiDriver,
String.valueOf(setupCsi));
k8sControlNodeConfig =
updateKubeConfigWithRegistryDetails(k8sControlNodeConfig);
@@ -246,7 +248,7 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
Long userDataId = kubernetesCluster.getCniConfigId();
Pair<String, String> k8sControlNodeConfigAndControlIp = new
Pair<>(null, null);
try {
- k8sControlNodeConfigAndControlIp =
getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName,
haSupported,
Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()),
Objects.nonNull(userDataId));
+ k8sControlNodeConfigAndControlIp =
getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName,
haSupported,
Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()),
Objects.nonNull(userDataId), kubernetesCluster.isCsiEnabled());
} catch (IOException e) {
logAndThrow(Level.ERROR, "Failed to read Kubernetes control node
configuration file", e);
}
@@ -858,6 +860,9 @@ public class KubernetesClusterStartWorker extends
KubernetesClusterResourceModif
}
taintControlNodes();
deployProvider();
+ if (kubernetesCluster.isCsiEnabled()) {
+ deployCsiDriver();
+ }
updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList()));
stateTransitTo(kubernetesCluster.getId(),
KubernetesCluster.Event.OperationSucceeded);
return true;
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java
b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java
index 4e92f9546f8..64d725797aa 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java
@@ -207,6 +207,9 @@ public class CreateKubernetesClusterCmd extends
BaseAsyncCreateCmd {
since = "4.21.0")
private Map cniConfigDetails;
+ @Parameter(name = ApiConstants.ENABLE_CSI, type = CommandType.BOOLEAN,
description = "if true, setups up CloudStack CSI driver", since = "4.21.0")
+ private Boolean enableCsi;
+
@Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG,
description="the AS Number of the network")
private Long asNumber;
@@ -371,6 +374,10 @@ public class CreateKubernetesClusterCmd extends
BaseAsyncCreateCmd {
return cniConfigId;
}
+ public boolean getEnableCsi() {
+ return Objects.nonNull(enableCsi) ? enableCsi : Boolean.FALSE;
+ }
+
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
diff --git
a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
index dc066e10d06..70291dd1c35 100644
---
a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
+++
b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
@@ -179,6 +179,11 @@ write_files:
mkdir -p /opt/provider
cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
fi
+ if [ -e "${BINARIES_DIR}/snapshot-crds.yaml" ]; then
+ mkdir -p /opt/csi
+ cp "${BINARIES_DIR}/snapshot-crds.yaml" /opt/csi/snapshot-crds.yaml
+ cp "${BINARIES_DIR}/manifest.yaml" /opt/csi/manifest.yaml
+ fi
PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail
-n 1`
echo $PAUSE_IMAGE
diff --git
a/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-csi-driver
b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-csi-driver
new file mode 100644
index 00000000000..dfc1ab48c97
--- /dev/null
+++
b/plugins/integrations/kubernetes-service/src/main/resources/script/deploy-csi-driver
@@ -0,0 +1,50 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+(/opt/bin/kubectl get pods -A | grep cloud-controller-manager) && exit 0
+
+if [ -e /opt/csi/snapshot-crds.yaml ]; then
+ /opt/bin/kubectl apply -f /opt/csi/snapshot-crds.yaml
+ sleep 5
+ /opt/bin/kubectl apply -f /opt/csi/manifest.yaml
+ exit 0
+else
+ TARGET_DIR="/opt/csi"
+ mkdir -p "$TARGET_DIR"
+# CSI_URLS=(
+#
"https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/snapshot-crds.yaml";
+#
"https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/manifest.yaml";
+# )
+ CSI_URLS=(
+ "http://10.0.3.130/cks/csi/snapshot-crds.yaml -O
${working_dir}/snapshot-crds.yaml"
+ "http://10.0.3.130/cks/csi/manifest.yaml -O
${working_dir}/manifest.yaml"
+ )
+ for url in "${URLS[@]}"; do
+ filename=$(basename "$url")
+
+ curl -sSL ${url} -o ${filename}
+ if [ $? -ne 0 ]; then
+ echo "Unable to connect to the internet to download the relevant
files to install and setup CloudStack CSI driver"
+ exit 1
+ else
+ /opt/bin/kubectl apply -f /opt/csi/snapshot-crds.yaml
+ /opt/bin/kubectl apply -f /opt/csi/manifest.yaml
+ exit 0
+ fi
+ done
+fi
\ No newline at end of file
diff --git a/scripts/util/create-kubernetes-binaries-iso.sh
b/scripts/util/create-kubernetes-binaries-iso.sh
index 535d4b36479..ef43fb620e3 100755
--- a/scripts/util/create-kubernetes-binaries-iso.sh
+++ b/scripts/util/create-kubernetes-binaries-iso.sh
@@ -156,6 +156,13 @@ if [ -n "${8}" ]; then
wget -q --show-progress
"https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz";
-O ${etcd_dir}/etcd-linux-amd64.tar.gz
fi
+echo "Including CloudStack CSI Driver manifest"
+# TODO: remove the below 2 lines and uncomment the ones below once csi-driver
is officially released, this is for testing purposes only
+wget http://10.0.3.130/cks/csi/snapshot-crds.yaml -O
${working_dir}/snapshot-crds.yaml
+wget http://10.0.3.130/cks/csi/manifest.yaml -O ${working_dir}/manifest.yaml
+#wget
https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/snapshot-crds.yaml
-O ${working_dir}/snapshot-crds.yaml
+#wget
https://github.com/shapeblue/cloudstack-csi-driver/releases/download/v3.0.0/manifest.yaml
-O ${working_dir}/manifest.yaml
+
mkisofs -o "${output_dir}/${build_name}" -J -R -l "${iso_dir}"
rm -rf "${iso_dir}"
diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json
index b649f6914bb..212947780d4 100644
--- a/ui/public/locales/en.json
+++ b/ui/public/locales/en.json
@@ -937,6 +937,7 @@
"label.elastic": "Elastic",
"label.email": "Email",
"label.enable.autoscale.vmgroup": "Enable AutoScaling Group",
+"label.enable.csi": "Enable CloudStack CSI Driver",
"label.enable.host": "Enable Host",
"label.enable.network.offering": "Enable Network offering",
"label.enable.oauth": "Enable OAuth Login",
diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue
b/ui/src/views/compute/CreateKubernetesCluster.vue
index 70cd06a20dd..684d602e7ff 100644
--- a/ui/src/views/compute/CreateKubernetesCluster.vue
+++ b/ui/src/views/compute/CreateKubernetesCluster.vue
@@ -207,6 +207,12 @@
</template>
<a-switch v-model:checked="form.advancedmode" />
</a-form-item>
+ <a-form-item v-if="form.advancedmode" name="enablecsi" ref="enablecsi"
:label="$t('label.enable.csi')">
+ <template #label>
+ <tooltip-label :title="$t('label.enable.csi')"
:tooltip="apiParams.enablecsi.description"/>
+ </template>
+ <a-switch v-model:checked="form.enablecsi" />
+ </a-form-item>
<a-form-item v-if="form.advancedmode" name="controlofferingid"
ref="controlofferingid">
<template #label>
<tooltip-label
:title="$t('label.cks.cluster.control.nodes.offeringid')"
:tooltip="$t('label.cks.cluster.control.nodes.offeringid')"/>
@@ -902,6 +908,10 @@ export default {
params.cniconfigurationid = values.cniconfigurationid
}
+ if (values.enablecsi) {
+ params.enablecsi = values.enablecsi
+ }
+
var idx = 0
if (this.cniConfigValues) {
for (const [key, value] of Object.entries(this.cniConfigValues)) {
