This is an automated email from the ASF dual-hosted git repository. wanghailin pushed a commit to branch dev in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push: new a1c90015d8 [Feature][K8S deploy] add helm chart (#8291) a1c90015d8 is described below commit a1c90015d8725ca1170c74aa6c118002ed36de47 Author: Daniel Duan <duanfangwei2...@sina.com> AuthorDate: Tue Dec 24 19:16:30 2024 +0800 [Feature][K8S deploy] add helm chart (#8291) --- .github/workflows/publish-helm-chart.yaml | 54 +++++++ deploy/kubernetes/seatunnel/Chart.yaml | 42 ++++++ .../seatunnel/conf/hazelcast-client.yaml | 27 ++++ .../seatunnel/conf/hazelcast-master.yaml | 46 ++++++ .../seatunnel/conf/hazelcast-worker.yaml | 50 +++++++ deploy/kubernetes/seatunnel/conf/log4j2.properties | 103 +++++++++++++ deploy/kubernetes/seatunnel/conf/seatunnel.yaml | 45 ++++++ deploy/kubernetes/seatunnel/templates/NOTES.txt | 66 +++++++++ deploy/kubernetes/seatunnel/templates/_helpers.tpl | 65 +++++++++ .../kubernetes/seatunnel/templates/configmap.yaml | 35 +++++ .../templates/deployment-seatunnel-master.yaml | 94 ++++++++++++ .../templates/deployment-seatunnel-worker.yaml | 91 ++++++++++++ deploy/kubernetes/seatunnel/templates/ingress.yaml | 60 ++++++++ deploy/kubernetes/seatunnel/templates/rbac.yaml | 53 +++++++ .../seatunnel/templates/service-headless.yaml | 33 +++++ .../templates/service-master-headless.yaml | 35 +++++ deploy/kubernetes/seatunnel/values.yaml | 160 +++++++++++++++++++++ docs/en/start-v2/kubernetes/helm.md | 80 +++++++++++ docs/zh/start-v2/kubernetes/helm.md | 80 +++++++++++ 19 files changed, 1219 insertions(+) diff --git a/.github/workflows/publish-helm-chart.yaml b/.github/workflows/publish-helm-chart.yaml new file mode 100644 index 0000000000..7eb2b2aaa8 --- /dev/null +++ b/.github/workflows/publish-helm-chart.yaml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: publish-helm-chart + +on: + push: + tags: + - '*' + paths-ignore: + - 'docs/**' + - '**/*.md' + +env: + DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKER_REGISTRY: docker.io + HUB: registry-1.docker.io/apache + +jobs: + build: + if: github.repository == 'apache/seatunnel' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.DOCKER_REGISTRY }} + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} + - name: Publish Helm Chart + working-directory: deploy/kubernetes + run: | + helm dep up seatunnel + helm package seatunnel + helm push seatunnel-helm-*.tgz oci://${{ env.HUB }} diff --git a/deploy/kubernetes/seatunnel/Chart.yaml b/deploy/kubernetes/seatunnel/Chart.yaml new file mode 100644 index 0000000000..dc843bef76 --- /dev/null +++ b/deploy/kubernetes/seatunnel/Chart.yaml @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +apiVersion: v2 +name: seatunnel-helm +description: SeaTunnel is a next-generation, high-performance, distributed data integration tool, capable of synchronizing vast amounts of data daily. It's trusted by numerous companies for its efficiency and stability. +home: https://seatunnel.apache.org +icon: https://seatunnel.apache.org/image/logo.png +keywords: + - seatunnel + - integration +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 2.3.9 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 2.3.9 \ No newline at end of file diff --git a/deploy/kubernetes/seatunnel/conf/hazelcast-client.yaml b/deploy/kubernetes/seatunnel/conf/hazelcast-client.yaml new file mode 100644 index 0000000000..ebd18db6b9 --- /dev/null +++ b/deploy/kubernetes/seatunnel/conf/hazelcast-client.yaml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +hazelcast-client: + cluster-name: {{ include "seatunnel.fullname" . }} + properties: + hazelcast.logging.type: log4j2 + connection-strategy: + connection-retry: + cluster-connect-timeout-millis: 3000 + network: + cluster-members: + - {{ include "seatunnel.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:5801 \ No newline at end of file diff --git a/deploy/kubernetes/seatunnel/conf/hazelcast-master.yaml b/deploy/kubernetes/seatunnel/conf/hazelcast-master.yaml new file mode 100644 index 0000000000..3f1686cef8 --- /dev/null +++ b/deploy/kubernetes/seatunnel/conf/hazelcast-master.yaml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +hazelcast: + cluster-name: {{ include "seatunnel.fullname" . }} + network: + rest-api: + enabled: true + endpoint-groups: + CLUSTER_WRITE: + enabled: true + DATA: + enabled: true + join: + kubernetes: + enabled: true + service-dns: {{ include "seatunnel.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local + service-port: 5801 + port: + auto-increment: false + port: 5801 + properties: + hazelcast.invocation.max.retry.count: 20 + hazelcast.tcp.join.port.try.count: 30 + hazelcast.logging.type: log4j2 + hazelcast.operation.generic.thread.count: 50 + hazelcast.heartbeat.failuredetector.type: phi-accrual + hazelcast.heartbeat.interval.seconds: 2 + hazelcast.max.no.heartbeat.seconds: 180 + hazelcast.heartbeat.phiaccrual.failuredetector.threshold: 10 + hazelcast.heartbeat.phiaccrual.failuredetector.sample.size: 200 + hazelcast.heartbeat.phiaccrual.failuredetector.min.std.dev.millis: 100 \ No newline at end of file diff --git a/deploy/kubernetes/seatunnel/conf/hazelcast-worker.yaml b/deploy/kubernetes/seatunnel/conf/hazelcast-worker.yaml new file mode 100644 index 0000000000..6d34ac29f2 --- /dev/null +++ b/deploy/kubernetes/seatunnel/conf/hazelcast-worker.yaml @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +hazelcast: + cluster-name: {{ include "seatunnel.fullname" . }} + network: + rest-api: + enabled: true + endpoint-groups: + CLUSTER_WRITE: + enabled: true + DATA: + enabled: true + join: + kubernetes: + enabled: true + service-dns: {{ include "seatunnel.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local + service-port: 5801 + port: + auto-increment: false + port: 5801 + properties: + hazelcast.invocation.max.retry.count: 20 + hazelcast.tcp.join.port.try.count: 30 + hazelcast.logging.type: log4j2 + hazelcast.operation.generic.thread.count: 50 + hazelcast.heartbeat.failuredetector.type: phi-accrual + hazelcast.heartbeat.interval.seconds: 2 + hazelcast.max.no.heartbeat.seconds: 180 + hazelcast.heartbeat.phiaccrual.failuredetector.threshold: 10 + hazelcast.heartbeat.phiaccrual.failuredetector.sample.size: 200 + hazelcast.heartbeat.phiaccrual.failuredetector.min.std.dev.millis: 100 + member-attributes: + rule: + type: string + value: worker \ No newline at end of file diff --git a/deploy/kubernetes/seatunnel/conf/log4j2.properties b/deploy/kubernetes/seatunnel/conf/log4j2.properties new file mode 100644 index 0000000000..9dd95de9e7 --- /dev/null +++ b/deploy/kubernetes/seatunnel/conf/log4j2.properties @@ -0,0 +1,103 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# The minimum amount of time, in seconds, that must elapse before the file configuration is checked for changes. +monitorInterval = 60 + +property.file_path = ${sys:seatunnel.logs.path:-/tmp/seatunnel/logs} +property.file_name = ${sys:seatunnel.logs.file_name:-seatunnel} +property.file_split_size = 100MB +property.file_count = 100 +property.file_ttl = 7d + +rootLogger.level = INFO + +logger.zeta.name=org.apache.seatunnel.engine +logger.zeta.level=INFO + +logger.debezium.name=io.debezium.connector +logger.debezium.level=WARN + +############################ log output to console ############################# +rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender +rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender +############################ log output to console ############################# +############################ log output to file ############################# +#rootLogger.appenderRef.file.ref = fileAppender +############################ log output to file ############################# + +appender.consoleStdout.name = consoleStdoutAppender +appender.consoleStdout.type = CONSOLE +appender.consoleStdout.target = SYSTEM_OUT +appender.consoleStdout.layout.type = PatternLayout +appender.consoleStdout.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n +appender.consoleStdout.filter.acceptLtWarn.type = ThresholdFilter +appender.consoleStdout.filter.acceptLtWarn.level = WARN +appender.consoleStdout.filter.acceptLtWarn.onMatch = DENY +appender.consoleStdout.filter.acceptLtWarn.onMismatch = ACCEPT + +appender.consoleStderr.name = consoleStderrAppender +appender.consoleStderr.type = CONSOLE +appender.consoleStderr.target = SYSTEM_ERR +appender.consoleStderr.layout.type = PatternLayout +appender.consoleStderr.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n +appender.consoleStderr.filter.acceptGteWarn.type = ThresholdFilter +appender.consoleStderr.filter.acceptGteWarn.level = WARN +appender.consoleStderr.filter.acceptGteWarn.onMatch = ACCEPT +appender.consoleStderr.filter.acceptGteWarn.onMismatch = DENY + +appender.routing.name = routingAppender +appender.routing.type = Routing +appender.routing.purge.type = IdlePurgePolicy +appender.routing.purge.timeToLive = 60 +appender.routing.route.type = Routes +appender.routing.route.pattern = $${ctx:ST-JID} +appender.routing.route.system.type = Route +appender.routing.route.system.key = $${ctx:ST-JID} +appender.routing.route.system.ref = fileAppender +appender.routing.route.job.type = Route +appender.routing.route.job.appender.type = File +appender.routing.route.job.appender.name = job-${ctx:ST-JID} +appender.routing.route.job.appender.fileName = ${file_path}/job-${ctx:ST-JID}.log +appender.routing.route.job.appender.layout.type = PatternLayout +appender.routing.route.job.appender.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n + +appender.file.name = fileAppender +appender.file.type = RollingFile +appender.file.fileName = ${file_path}/${file_name}.log +appender.file.filePattern = ${file_path}/${file_name}.log.%d{yyyy-MM-dd}-%i +appender.file.append = true +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n +appender.file.policies.type = Policies +appender.file.policies.time.type = TimeBasedTriggeringPolicy +appender.file.policies.time.modulate = true +appender.file.policies.size.type = SizeBasedTriggeringPolicy +appender.file.policies.size.size = ${file_split_size} +appender.file.strategy.type = DefaultRolloverStrategy +appender.file.strategy.fileIndex = nomax +appender.file.strategy.action.type = Delete +appender.file.strategy.action.basepath = ${file_path} +appender.file.strategy.action.maxDepth = 1 +appender.file.strategy.action.condition.type = IfFileName +appender.file.strategy.action.condition.glob = ${file_name}.log* +appender.file.strategy.action.condition.nested_condition.type = IfAny +appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified +appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl} +appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount +appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count} \ No newline at end of file diff --git a/deploy/kubernetes/seatunnel/conf/seatunnel.yaml b/deploy/kubernetes/seatunnel/conf/seatunnel.yaml new file mode 100644 index 0000000000..e91fd50933 --- /dev/null +++ b/deploy/kubernetes/seatunnel/conf/seatunnel.yaml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +seatunnel: + engine: + history-job-expire-minutes: 1440 + backup-count: 1 + queue-type: blockingqueue + print-execution-info-interval: 60 + print-job-metrics-info-interval: 60 + classloader-cache-mode: true + slot-service: + dynamic-slot: true + http: + enable-http: true + port: 8080 + enable-dynamic-port: false + port-range: 100 + checkpoint: + interval: 300000 + timeout: 10000 + storage: + type: hdfs + max-retained: 3 + plugin-config: + namespace: /tmp/seatunnel/checkpoint_snapshot/ + storage.type: hdfs + fs.defaultFS: file:///tmp/ + telemetry: + metric: + enabled: true diff --git a/deploy/kubernetes/seatunnel/templates/NOTES.txt b/deploy/kubernetes/seatunnel/templates/NOTES.txt new file mode 100644 index 0000000000..d06fe5d3ba --- /dev/null +++ b/deploy/kubernetes/seatunnel/templates/NOTES.txt @@ -0,0 +1,66 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +** Please be patient while the chart seatunnel {{ .Chart.AppVersion }} is being deployed ** + +Access seatunnel UI URL by: + +{{- if .Values.ingress.enabled }} + + seatunnel restapi URL for running jobs: http{{ if .Values.ingress.tls.enabled }}s{{ end }}://{{ .Values.ingress.host }}/running-jobs + seatunnel restapi URL for system monitoring information: http{{ if .Values.ingress.tls.enabled }}s{{ end }}://{{ .Values.ingress.host }}/system-monitoring-information + + For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2 + +{{- else if eq .Values.master.service.type "ClusterIP" }} + + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ template "seatunnel.fullname" . }}-master 8080:8080 + + seatunnel restapi URL for running jobs: http://127.0.0.1:8080/running-jobs + seatunnel restapi URL for system monitoring information: http://127.0.0.1:8080/system-monitoring-information + + For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2 + + +{{- else if eq .Values.master.service.type "NodePort" }} + + NODE_IP=$(kubectl get no -n {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + NODE_PORT=$(kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}") + seatunnel restapi URL for running jobs: http://$NODE_IP:$NODE_PORT/running-jobs + seatunnel restapi URL for system monitoring information: http://$NODE_IP:$NODE_PORT/system-monitoring-information + + For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2 +{{- else if eq .Values.master.service.type "LoadBalancer" }} + + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -w' + + SERVICE_IP=$(kubectl get svc {{ template "seatunnel.fullname" . }}-master -n {{ .Release.Namespace }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:8080/running-jobs + + seatunnel resetapi URL for running jobs: http://$SERVICE_IP:8080/running-jobs + seatunnel resetapi URL for system monitoring information: http://$SERVICE_IP:8080/system-monitoring-information + + For more restapi please refer to https://seatunnel.apache.org/docs/{{ .Chart.AppVersion }}/seatunnel-engine/rest-api-v2 +{{- end }} + +Or you can just go into master pod, and use local curl command. + +MASTER_POD=$(kubectl get po -l 'app.kubernetes.io/name=seatunnel-master' | sed '1d' | awk '{print $1}' | head -n1) +kubectl -n {{ .Release.Namespace }} exec -it $MASTER_POD -- /bin/bash +curl http://127.0.0.1:8080/running-jobs +curl http://127.0.0.1:8080/system-monitoring-information \ No newline at end of file diff --git a/deploy/kubernetes/seatunnel/templates/_helpers.tpl b/deploy/kubernetes/seatunnel/templates/_helpers.tpl new file mode 100644 index 0000000000..9f7c0846b2 --- /dev/null +++ b/deploy/kubernetes/seatunnel/templates/_helpers.tpl @@ -0,0 +1,65 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{/* vim: set filetype=mustache: */}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "seatunnel.fullname" -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create default docker images' fullname. +*/}} +{{- define "seatunnel.image.fullname.master" -}} +{{- .Values.image.registry }}:{{ .Values.image.tag | default .Chart.AppVersion -}} +{{- end -}} +{{- define "seatunnel.image.fullname.worker" -}} +{{- .Values.image.registry }}:{{ .Values.image.tag | default .Chart.AppVersion -}} +{{- end -}} + +{{/* +Create a default common labels. +*/}} +{{- define "seatunnel.common.labels" -}} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/version: {{ .Chart.AppVersion }} + +{{- end -}} + +{{/* +Create a master labels. +*/}} +{{- define "seatunnel.master.labels" -}} +app.kubernetes.io/name: {{ include "seatunnel.fullname" . }}-master +app.kubernetes.io/component: master +{{ include "seatunnel.common.labels" . }} +{{- end -}} + +{{/* +Create a worker labels. +*/}} +{{- define "seatunnel.worker.labels" -}} +app.kubernetes.io/name: {{ include "seatunnel.fullname" . }}-worker +app.kubernetes.io/component: worker +{{ include "seatunnel.common.labels" . }} +{{- end -}} diff --git a/deploy/kubernetes/seatunnel/templates/configmap.yaml b/deploy/kubernetes/seatunnel/templates/configmap.yaml new file mode 100644 index 0000000000..5eface25f6 --- /dev/null +++ b/deploy/kubernetes/seatunnel/templates/configmap.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "seatunnel.fullname" . }}-configs + labels: + app.kubernetes.io/name: {{ include "seatunnel.fullname" . }}-configs + {{- include "seatunnel.master.labels" . | nindent 4 }} +data: + hazelcast-master.yaml: |- + {{ tpl (.Files.Get "conf/hazelcast-master.yaml") $ | nindent 4 }} + hazelcast-worker.yaml: |- + {{ tpl (.Files.Get "conf/hazelcast-worker.yaml") $ | nindent 4 }} + hazelcast-client.yaml: |- + {{ tpl (.Files.Get "conf/hazelcast-client.yaml") $ | nindent 4 }} + seatunnel.yaml: |- + {{ tpl (.Files.Get "conf/seatunnel.yaml") $ | nindent 4 }} + log4j2.properties: |- + {{ tpl (.Files.Get "conf/log4j2.properties") $ | nindent 4 }} + diff --git a/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml b/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml new file mode 100644 index 0000000000..517fc3ac53 --- /dev/null +++ b/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-master.yaml @@ -0,0 +1,94 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "seatunnel.fullname" . }}-master + labels: + {{- include "seatunnel.master.labels" . | nindent 4 }} +spec: + {{- if .Values.master.strategy }} + strategy: + {{- toYaml .Values.master.strategy | nindent 4 }} + {{- end }} + replicas: {{ .Values.master.replicas }} + selector: + matchLabels: + {{- include "seatunnel.master.labels" . | nindent 6 }} + template: + metadata: + {{- if .Values.master.annotations }} + annotations: + {{- toYaml .Values.master.annotations | nindent 8 }} + {{- end }} + labels: + {{- include "seatunnel.master.labels" . | nindent 8 }} + spec: + serviceAccountName: {{ template "seatunnel.fullname" . }} + {{- if .Values.master.affinity }} + affinity: + {{- toYaml .Values.master.affinity | nindent 8 }} + {{- end }} + {{- if .Values.master.nodeSelector }} + nodeSelector: + {{- toYaml .Values.master.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.master.tolerations }} + tolerations: + {{- toYaml .Values.master.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.image.pullSecret }} + imagePullSecrets: + - name: {{ .Values.image.pullSecret }} + {{- end }} + containers: + - name: {{ include "seatunnel.fullname" . }}-master + image: {{ include "seatunnel.image.fullname.master" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 5801 + name: "hazelcast-port" + - containerPort: 8080 + name: "master-port" + command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel-cluster.sh -r master"] + {{- if .Values.master.resources }} + resources: + {{- toYaml .Values.master.resources | nindent 12 }} + {{- end }} + {{- if .Values.master.livenessProbe.enabled }} + livenessProbe: + {{- toYaml .Values.master.livenessProbe | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: "/opt/seatunnel/config/hazelcast-master.yaml" + name: seatunnel-configs + subPath: hazelcast-master.yaml + - mountPath: "/opt/seatunnel/config/hazelcast-client.yaml" + name: seatunnel-configs + subPath: hazelcast-client.yaml + - mountPath: "/opt/seatunnel/config/seatunnel.yaml" + name: seatunnel-configs + subPath: seatunnel.yaml + - mountPath: "/opt/seatunnel/config/log4j2.properties" + name: seatunnel-configs + subPath: log4j2.properties + volumes: + - name: seatunnel-configs + configMap: + name: {{ include "seatunnel.fullname" . }}-configs + diff --git a/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-worker.yaml b/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-worker.yaml new file mode 100644 index 0000000000..d56263f908 --- /dev/null +++ b/deploy/kubernetes/seatunnel/templates/deployment-seatunnel-worker.yaml @@ -0,0 +1,91 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "seatunnel.fullname" . }}-worker + labels: + {{- include "seatunnel.worker.labels" . | nindent 4 }} +spec: + {{- if .Values.worker.strategy }} + strategy: + {{- toYaml .Values.worker.strategy | nindent 4 }} + {{- end }} + replicas: {{ .Values.worker.replicas }} + selector: + matchLabels: + {{- include "seatunnel.worker.labels" . | nindent 6 }} + template: + metadata: + {{- if .Values.worker.annotations }} + annotations: + {{- toYaml .Values.worker.annotations | nindent 8 }} + {{- end }} + labels: + {{- include "seatunnel.worker.labels" . | nindent 8 }} + spec: + serviceAccountName: {{ template "seatunnel.fullname" . }} + {{- if .Values.worker.affinity }} + affinity: + {{- toYaml .Values.worker.affinity | nindent 8 }} + {{- end }} + {{- if .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml .Values.worker.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.worker.tolerations }} + tolerations: + {{- toYaml .Values.worker.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.image.pullSecret }} + imagePullSecrets: + - name: {{ .Values.image.pullSecret }} + {{- end }} + containers: + - name: {{ include "seatunnel.fullname" . }}-worker + image: {{ include "seatunnel.image.fullname.worker" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 5801 + name: "hazelcast-port" + command: ["/bin/sh","-c","/opt/seatunnel/bin/seatunnel-cluster.sh -r worker"] + {{- if .Values.worker.resources }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + {{- end }} + {{- if .Values.worker.livenessProbe.enabled }} + livenessProbe: + {{- toYaml .Values.worker.livenessProbe | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: "/opt/seatunnel/config/hazelcast-worker.yaml" + name: seatunnel-configs + subPath: hazelcast-worker.yaml + - mountPath: "/opt/seatunnel/config/hazelcast-client.yaml" + name: seatunnel-configs + subPath: hazelcast-client.yaml + - mountPath: "/opt/seatunnel/config/seatunnel.yaml" + name: seatunnel-configs + subPath: seatunnel.yaml + - mountPath: "/opt/seatunnel/config/log4j2.properties" + name: seatunnel-configs + subPath: log4j2.properties + volumes: + - name: seatunnel-configs + configMap: + name: {{ include "seatunnel.fullname" . }}-configs diff --git a/deploy/kubernetes/seatunnel/templates/ingress.yaml b/deploy/kubernetes/seatunnel/templates/ingress.yaml new file mode 100644 index 0000000000..97319a956d --- /dev/null +++ b/deploy/kubernetes/seatunnel/templates/ingress.yaml @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +{{- if and .Values.ingress.enabled }} +{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} +apiVersion: networking.k8s.io/v1 +{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/Ingress" }} +apiVersion: networking.k8s.io/v1beta1 +{{- else }} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ include "seatunnel.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "seatunnel.fullname" . }} + {{- include "seatunnel.common.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + rules: + - host: "{{ .Values.ingress.host }}" + http: + paths: + - path: {{ .Values.ingress.path }} + backend: + {{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} + service: + name: {{ include "seatunnel.fullname" . }}-master + port: + number: 8080 + {{- else }} + serviceName: {{ include "seatunnel.fullname" . }}-master + servicePort: 8080 + {{- end }} + {{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} + pathType: Prefix + {{- end }} + {{- if .Values.ingress.tls.enabled }} + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ .Values.ingress.tls.secretName }} + {{- end }} +{{- end }} diff --git a/deploy/kubernetes/seatunnel/templates/rbac.yaml b/deploy/kubernetes/seatunnel/templates/rbac.yaml new file mode 100644 index 0000000000..6095b80893 --- /dev/null +++ b/deploy/kubernetes/seatunnel/templates/rbac.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "seatunnel.fullname" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + release: {{ .Release.Name }} + name: {{ template "seatunnel.fullname" . }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "seatunnel.fullname" . }} + labels: + app: {{ template "seatunnel.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "seatunnel.fullname" . }} + labels: + app: {{ template "seatunnel.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "seatunnel.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "seatunnel.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/kubernetes/seatunnel/templates/service-headless.yaml b/deploy/kubernetes/seatunnel/templates/service-headless.yaml new file mode 100644 index 0000000000..be31c5dee0 --- /dev/null +++ b/deploy/kubernetes/seatunnel/templates/service-headless.yaml @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# use for hazelcast cluster join +apiVersion: v1 +kind: Service +metadata: + name: {{ include "seatunnel.fullname" . }} + labels: + {{- include "seatunnel.common.labels" . | nindent 4 }} + namespace: {{ .Values.namespace }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: "hazelcast-port" + port: 5801 + selector: + {{- include "seatunnel.common.labels" . | nindent 4 }} diff --git a/deploy/kubernetes/seatunnel/templates/service-master-headless.yaml b/deploy/kubernetes/seatunnel/templates/service-master-headless.yaml new file mode 100644 index 0000000000..63aff3ba75 --- /dev/null +++ b/deploy/kubernetes/seatunnel/templates/service-master-headless.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--- +# use for access seatunnel from outside system via rest api +apiVersion: v1 +kind: Service +metadata: + name: {{ include "seatunnel.fullname" . }}-master + labels: + {{- include "seatunnel.master.labels" . | nindent 4 }} + namespace: {{ .Values.namespace }} +spec: + clusterIP: "None" + ports: + - name: "master-port" + port: 8080 + targetPort: 8080 + protocol: TCP + selector: + {{- include "seatunnel.master.labels" . | nindent 4 }} +--- diff --git a/deploy/kubernetes/seatunnel/values.yaml b/deploy/kubernetes/seatunnel/values.yaml new file mode 100644 index 0000000000..b5fd60b750 --- /dev/null +++ b/deploy/kubernetes/seatunnel/values.yaml @@ -0,0 +1,160 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default values for seatunnel-chart. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +timezone: "Asia/Shanghai" + + +image: + registry: "apache/seatunnel" + tag: "" + pullPolicy: "IfNotPresent" + pullSecret: "" + +master: + ## The deployment strategy to use to replace existing pods with new ones. + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 50% + + ## Replicas is the desired number of replicas of the given Template. + replicas: "2" + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + annotations: + prometheus.io/path: /hazelcast/rest/instance/metrics + prometheus.io/port: "5801" + prometheus.io/scrape: "true" + prometheus.io/role: "seatunnel-master" + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: {} + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + tolerations: [] + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container + resources: {} + # resources: + # limits: + # memory: "4Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + + ## service config for restApi + service: + type: "ClusterIP" + + livenessProbe: + tcpSocket: + port: hazelcast-port + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + +worker: + ## The deployment strategy to use to replace existing pods with new ones. + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 50% + + ## Replicas is the desired number of replicas of the given Template. + replicas: "2" + ## You can use annotations to attach arbitrary non-identifying metadata to objects. + ## Clients such as tools and libraries can retrieve this metadata. + ## Add enable prometheus scrape for metrics collection. + annotations: + prometheus.io/path: /hazelcast/rest/instance/metrics + prometheus.io/port: "5801" + prometheus.io/scrape: "true" + prometheus.io/role: "seatunnel-worker" + ## Affinity is a group of affinity scheduling rules. If specified, the pod's scheduling constraints. + ## More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#affinity-v1-core + affinity: {} + ## NodeSelector is a selector which must be true for the pod to fit on a node. + ## Selector which must match a node's labels for the pod to be scheduled on that node. + ## More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + nodeSelector: {} + ## Tolerations are appended (excluding duplicates) to pods running with this RuntimeClass during admission, + ## effectively unioning the set of nodes tolerated by the pod and the RuntimeClass. + tolerations: [] + ## Compute Resources required by this container. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container + resources: {} + # resources: + # limits: + # memory: "4Gi" + # cpu: "4" + # requests: + # memory: "2Gi" + # cpu: "500m" + ## Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + + + livenessProbe: + tcpSocket: + port: hazelcast-port + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + ## Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. + ## More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + readinessProbe: + enabled: true + initialDelaySeconds: "30" + periodSeconds: "30" + timeoutSeconds: "5" + failureThreshold: "3" + successThreshold: "1" + +ingress: + enabled: false + host: "seatunnel.org" + path: "/" + annotations: {} + tls: + enabled: false + secretName: "seatunnel-tls" diff --git a/docs/en/start-v2/kubernetes/helm.md b/docs/en/start-v2/kubernetes/helm.md new file mode 100644 index 0000000000..eebfda3c97 --- /dev/null +++ b/docs/en/start-v2/kubernetes/helm.md @@ -0,0 +1,80 @@ +--- +sidebar_position: 4 +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Set Up with Helm + +This section provides a quick guide to use SeaTunnel with Helm. + +## Prerequisites + +We assume that you have one local installation as follow: + +- [docker](https://docs.docker.com/) +- [kubernetes](https://kubernetes.io/) +- [helm](https://helm.sh/docs/intro/quickstart/) + +So that the `kubectl` and `helm` commands are available on your local system. + +Take kubernetes [minikube](https://minikube.sigs.k8s.io/docs/start/) as an example, you can start a cluster with the following command: + +```bash +minikube start --kubernetes-version=v1.23.3 +``` + +## Install + +Install with default settings. +```bash +# Choose the corresponding version yourself +export VERSION=2.3.9 +helm pull oci://registry-1.docker.io/apache/seatunnel-helm --version ${VERSION} +tar -xvf seatunnel-helm-${VERSION}.tgz +cd seatunnel-helm +helm install seatunnel . +``` +Install with another namespace. +```bash +helm install seatunnel . -n <your namespace> +``` + +## Submit Job + +The default config doesn't enable ingress, so you need forward the master restapi. +```bash +kubectl port-forward -n default svc/seatunnel-master 5801:5801 +``` +Then you can access restapi with "http://127.0.0.1/5801/" + +If you want to use ingress, update `value.yaml` + +for example: +```commandline +ingress: + enabled: true + host: "<your domain>" +``` +Then upgrade seatunnel. + +Then you can access restapi with `http://<your domain>` + +Or you can just go into master pod, and use local curl command. +```commandline +# get one of the master pods +MASTER_POD=$(kubectl get po -l 'app.kubernetes.io/name=seatunnel-master' | sed '1d' | awk '{print $1}' | head -n1) +# go into master pod container. +kubectl -n default exec -it $MASTER_POD -- /bin/bash + +curl http://127.0.0.1:5801/running-jobs +curl http://127.0.0.1:5801/system-monitoring-information +``` + +After that you can submit your job by [rest-api-v2](../../seatunnel-engine/rest-api-v2.md) + +## What's More + +For now, you have taken a quick look at SeaTunnel, and you can see [connector](../../connector-v2) to find all sources and sinks SeaTunnel supported. +Or see [deployment](../../seatunnel-engine/deployment.md) if you want to submit your application in another kind of your engine cluster. diff --git a/docs/zh/start-v2/kubernetes/helm.md b/docs/zh/start-v2/kubernetes/helm.md new file mode 100644 index 0000000000..e0d6c60aca --- /dev/null +++ b/docs/zh/start-v2/kubernetes/helm.md @@ -0,0 +1,80 @@ +--- +sidebar_position: 4 +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 使用Helm部署 + +使用Helm快速部署Seatunnel集群。 + +## 准备 + +我们假设您的本地已经安装如下软件: + +- [docker](https://docs.docker.com/) +- [kubernetes](https://kubernetes.io/) +- [helm](https://helm.sh/docs/intro/quickstart/) + +在您的本地环境中能够正常执行`kubectl`和`helm`命令。 + +以 [minikube](https://minikube.sigs.k8s.io/docs/start/) 为例, 您可以使用如下命令启动一个集群: + +```bash +minikube start --kubernetes-version=v1.23.3 +``` + +## 安装 + +使用默认配置安装 +```bash +# Choose the corresponding version yourself +export VERSION=2.3.9 +helm pull oci://registry-1.docker.io/apache/seatunnel-helm --version ${VERSION} +tar -xvf seatunnel-helm-${VERSION}.tgz +cd seatunnel-helm +helm install seatunnel . +``` + +如果您需要使用其他命名空间进行安装。 +``` +helm install seatunnel . -n <your namespace> +``` + +## 提交任务 + +当前默认的配置没有启用ingress,所以需要使用转发命令将master的restapi端口转发出来。 +```bash +kubectl port-forward -n default svc/seatunnel-master 5801:5801 +``` +然后可以通过地址 "http://127.0.0.1/5801/" 访问master的restapi。 + +如果想要使用ingress, 需要更新 `value.yaml` + +例如: +```commandline +ingress: + enabled: true + host: "<your domain>" +``` +然后更新seatunnel。 + +就可以使用域名`http://<your domain>`进行访问了。 + +或者您可以直接进入master的POD执行curl命令。. +```commandline +# 获取其中一个master pod +MASTER_POD=$(kubectl get po -l 'app.kubernetes.io/name=seatunnel-master' | sed '1d' | awk '{print $1}' | head -n1) +# 进入master pod +kubectl -n default exec -it $MASTER_POD -- /bin/bash +# 执行 restapi +curl http://127.0.0.1:5801/running-jobs +curl http://127.0.0.1:5801/system-monitoring-information +``` + +后面就可以使用[rest-api-v2](../../seatunnel-engine/rest-api-v2.md)提交任务了。 + +## 下一步 +到现在为止,您已经安装好Seatunnel集群了,你可以查看Seatunnel有哪些[连接器](../../connector-v2). +或者选择其他方式 [部署](../../seatunnel-engine/deployment.md).