This is an automated email from the ASF dual-hosted git repository.
wwei pushed a commit to branch soak-test
in repository https://gitbox.apache.org/repos/asf/yunikorn-release.git
The following commit(s) were added to refs/heads/soak-test by this push:
new 3a04994 [YUNIKORN-3010] Environment setup scripts that installs Kind,
Kwok, YuniKorn (#187)
3a04994 is described below
commit 3a0499434067c072dd71ad2f5497952a043d9f02
Author: junyan-ling <[email protected]>
AuthorDate: Wed Feb 12 20:47:24 2025 -0800
[YUNIKORN-3010] Environment setup scripts that installs Kind, Kwok,
YuniKorn (#187)
This PR adds the init script for setting up the soak testing environment,
that installs Kind, Kwok and YuniKorn.
Co-authored-by: Junyan Ling <[email protected]>
---
soak/pkg/constants/constants.go | 23 +++++
soak/pkg/setup/setup.go | 174 +++++++++++++++++++++++++++++++++
soak/scripts/README.md | 28 ++++++
soak/scripts/initial_setup.sh | 32 ++++++
soak/templates/autoscaling-queues.yaml | 37 +++++++
soak/templates/kwok-node-template.yaml | 58 +++++++++++
6 files changed, 352 insertions(+)
diff --git a/soak/pkg/constants/constants.go b/soak/pkg/constants/constants.go
new file mode 100644
index 0000000..0ee39c2
--- /dev/null
+++ b/soak/pkg/constants/constants.go
@@ -0,0 +1,23 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package constants
+
+const (
+ KindSoakTestCluster = "kind-soak-test-cluster"
+)
diff --git a/soak/pkg/setup/setup.go b/soak/pkg/setup/setup.go
new file mode 100644
index 0000000..307d170
--- /dev/null
+++ b/soak/pkg/setup/setup.go
@@ -0,0 +1,174 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package setup
+
+import (
+ "fmt"
+ "github.com/apache/yunikorn-core/pkg/log"
+ "github.com/apache/yunikorn-release/soak/pkg/constants"
+ "github.com/apache/yunikorn-release/soak/pkg/framework"
+ "go.uber.org/zap"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+)
+
+var logger *zap.Logger = log.Log(log.Test)
+
+func setK8sContext() error {
+ homeDir, err := os.UserHomeDir()
+ if err != nil {
+ return fmt.Errorf("failed to get home directory: %v", err)
+ }
+ kubeconfigPath := filepath.Join(homeDir, ".kube", "config")
+ os.Setenv("KUBECONFIG", kubeconfigPath)
+ logger.Info("Set KUBECONFIG", zap.String("path", kubeconfigPath))
+
+ contextCmd := exec.Command("kubectl", "config", "use-context",
constants.KindSoakTestCluster)
+ contextOutput, err := contextCmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("failed to switch kubectl context: %v,
output: %s", err, string(contextOutput))
+ }
+ logger.Info("Kubectl context switch output", zap.String("output",
strings.TrimSpace(string(contextOutput))))
+
+ currentContextCmd := exec.Command("kubectl", "config",
"current-context")
+ _, err = currentContextCmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("failed to get current context: %v", err)
+ }
+
+ return nil
+}
+
+func upgradeSchedulerPerConfig(scheduler framework.TemplateFields) error {
+ if err := setK8sContext(); err != nil {
+ logger.Fatal("failed to set kubernetes context", zap.Error(err))
+ return err
+ }
+
+ logger.Info("Scheduler details",
+ zap.String("VcoreRequests", *scheduler.VcoreRequests),
+ zap.String("MemoryRequests", *scheduler.MemoryRequests),
+ zap.String("VcoreLimits", *scheduler.VcoreLimits),
+ zap.String("MemoryLimits", *scheduler.MemoryLimits),
+ zap.String("path", *scheduler.Path))
+
+ args := []string{
+ "upgrade",
+ "yunikorn",
+ "yunikorn/yunikorn",
+ "-n", "yunikorn",
+ }
+
+ var moreArgs []string
+
+ if scheduler.VcoreRequests != nil {
+ moreArgs = append(moreArgs, "--set",
fmt.Sprintf("resources.requests.cpu=%s", *scheduler.VcoreRequests))
+ }
+ if scheduler.MemoryRequests != nil {
+ moreArgs = append(moreArgs, "--set",
fmt.Sprintf("resources.requests.memory=%s", *scheduler.MemoryRequests))
+ }
+ if scheduler.VcoreLimits != nil {
+ moreArgs = append(moreArgs, "--set",
fmt.Sprintf("resources.limits.cpu=%s", *scheduler.VcoreLimits))
+ }
+ if scheduler.MemoryLimits != nil {
+ moreArgs = append(moreArgs, "--set",
fmt.Sprintf("resources.limits.memory=%s", *scheduler.MemoryLimits))
+ }
+
+ if len(moreArgs) > 0 {
+ args = append(args, moreArgs...)
+
+ cmd := exec.Command("helm", args...)
+
+ logger.Info("Helm command to be executed",
+ zap.String("command", fmt.Sprintf("helm %s",
strings.Join(args, " "))))
+
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("helm upgrade failed: %v", err)
+ }
+
+ logger.Info("Helm upgrade successful",
+ zap.String("command", fmt.Sprintf("helm %s",
strings.Join(args, " "))),
+ zap.String("output", string(output)))
+ }
+
+ if scheduler.Path != nil {
+ kubectlArgs := []string{"apply"}
+ kubectlArgs = append(kubectlArgs, "-f", *scheduler.Path, "-n",
"yunikorn")
+ kubectlCmd := exec.Command("kubectl", kubectlArgs...)
+ logger.Info("Kubectl command to be executed",
+ zap.String("command", fmt.Sprintf("kubectl %s",
strings.Join(kubectlArgs, " "))))
+
+ kubectlOutput, err := kubectlCmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("kubectl apply failed: %v", err)
+ }
+ logger.Info("Kubectl apply successful", zap.String("output",
strings.TrimSpace(string(kubectlOutput))))
+ }
+
+ return nil
+}
+
+func setNodeScalePerConfig(node framework.TemplateFields) error {
+ if err := setK8sContext(); err != nil {
+ logger.Fatal("failed to set kubernetes context", zap.Error(err))
+ return err
+ }
+
+ logger.Info("Node details",
+ zap.String("path", *node.Path),
+ zap.Int("NodesDesiredCount", *node.DesiredCount),
+ zap.Int("maxCount", *node.MaxCount))
+
+ templateContent, err :=
os.ReadFile("soak/templates/kwok-node-template.yaml")
+ if err != nil {
+ return fmt.Errorf("failed to read template file: %v", err)
+ }
+ desiredCount := *node.DesiredCount
+
+ for i := 0; i < desiredCount; i++ {
+ currentNodeName := fmt.Sprintf("kwok-node-%d", i)
+ nodeContent := strings.ReplaceAll(string(templateContent),
"kwok-node-i", currentNodeName)
+
+ tmpfile, err := os.CreateTemp("", "node-*.yaml")
+ if err != nil {
+ return fmt.Errorf("failed to create temp file: %v", err)
+ }
+ defer os.Remove(tmpfile.Name()) // Clean up
+
+ if _, err := tmpfile.WriteString(nodeContent); err != nil {
+ return fmt.Errorf("failed to write to temp file: %v",
err)
+ }
+ if err := tmpfile.Close(); err != nil {
+ return fmt.Errorf("failed to close temp file: %v", err)
+ }
+
+ cmd := exec.Command("kubectl", "apply", "-f", tmpfile.Name())
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("failed to apply node configuration:
%v", err)
+ }
+
+ logger.Info("Applied node configuration",
+ zap.String("nodeName", currentNodeName),
+ zap.String("output", string(output)))
+ }
+
+ return nil
+}
diff --git a/soak/scripts/README.md b/soak/scripts/README.md
new file mode 100644
index 0000000..96f58f7
--- /dev/null
+++ b/soak/scripts/README.md
@@ -0,0 +1,28 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+### Prerequisites
+- [Helm](https://github.com/helm/helm#install)
+- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
+- [Docker](https://docs.docker.com/get-docker/)
+- [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)
+- [Kwok](https://kwok.sigs.k8s.io/docs/user/installation/)
+
+# set up initial soak test environment
+```
+sh soak/setup/initial_setup.sh
+```
diff --git a/soak/scripts/initial_setup.sh b/soak/scripts/initial_setup.sh
new file mode 100644
index 0000000..4b4bc33
--- /dev/null
+++ b/soak/scripts/initial_setup.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# create a kind cluster
+kind create cluster --name soak-test-cluster
+
+# install YuniKorn scheduler on kind Cluster
+helm repo add yunikorn https://apache.github.io/yunikorn-release
+helm repo update
+kubectl create namespace yunikorn
+# TODO: allow to install a customized YuniKorn version to run the soak test
+helm install yunikorn yunikorn/yunikorn --namespace yunikorn
+
+## Deploy kwok in a Cluster
+helm repo add kwok https://kwok.sigs.k8s.io/charts/
+helm upgrade --namespace kube-system --install kwok kwok/kwok
+helm upgrade --install kwok kwok/stage-fast
diff --git a/soak/templates/autoscaling-queues.yaml
b/soak/templates/autoscaling-queues.yaml
new file mode 100644
index 0000000..89bafdd
--- /dev/null
+++ b/soak/templates/autoscaling-queues.yaml
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: yunikorn-defaults
+ namespace: yunikorn
+data:
+ queues.yaml: |-
+ partitions:
+ - name: default
+ queues:
+ - name: root
+ submitacl: '*'
+ queues:
+ - name: queue-a
+ submitacl: '*'
+ properties:
+ application.sort.policy: fair
+ - name: queue-b
+ submitacl: '*'
+ properties:
+ application.sort.policy: fifo
diff --git a/soak/templates/kwok-node-template.yaml
b/soak/templates/kwok-node-template.yaml
new file mode 100644
index 0000000..5f1de38
--- /dev/null
+++ b/soak/templates/kwok-node-template.yaml
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Node
+metadata:
+ annotations:
+ node.alpha.kubernetes.io/ttl: "0"
+ kwok.x-k8s.io/node: fake
+ labels:
+ beta.kubernetes.io/arch: amd64
+ beta.kubernetes.io/os: linux
+ kubernetes.io/arch: amd64
+ kubernetes.io/hostname: kwok-node-i
+ kubernetes.io/os: linux
+ kubernetes.io/role: agent
+ node-role.kubernetes.io/agent: ""
+ type: kwok
+ name: kwok-node-i
+spec:
+ taints: # Avoid scheduling actual running pods to fake Node
+ - effect: NoSchedule
+ key: kwok.x-k8s.io/node
+ value: fake
+status:
+ allocatable:
+ cpu: 32
+ memory: 256Gi
+ pods: 110
+ capacity:
+ cpu: 32
+ memory: 256Gi
+ pods: 110
+ nodeInfo:
+ architecture: amd64
+ bootID: ""
+ containerRuntimeVersion: ""
+ kernelVersion: ""
+ kubeProxyVersion: fake
+ kubeletVersion: fake
+ machineID: ""
+ operatingSystem: linux
+ osImage: ""
+ systemUUID: ""
+ phase: Running
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
