imbajin commented on code in PR #3132: URL: https://github.com/apache/hugegraph/pull/3132#discussion_r3703015612
########## helm/hugegraph/templates/_helpers.tpl: ########## @@ -0,0 +1,295 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{/* +Expand the name of the chart. +*/}} +{{- define "hugegraph.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "hugegraph.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "hugegraph.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "hugegraph.labels" -}} +helm.sh/chart: {{ include "hugegraph.chart" . }} +{{ include "hugegraph.selectorLabels" . }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "hugegraph.selectorLabels" -}} +app.kubernetes.io/name: {{ include "hugegraph.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "hugegraph.pd.name" -}} +{{- printf "%s-pd" (include "hugegraph.fullname" . | trunc 57 | trimSuffix "-") }} +{{- end }} + +{{- define "hugegraph.pd.clientName" -}} +{{- printf "%s-pd-client" (include "hugegraph.fullname" . | trunc 53 | trimSuffix "-") }} +{{- end }} + +{{- define "hugegraph.store.name" -}} +{{- printf "%s-store" (include "hugegraph.fullname" . | trunc 54 | trimSuffix "-") }} +{{- end }} + +{{- define "hugegraph.server.name" -}} +{{- printf "%s-server" (include "hugegraph.fullname" . | trunc 56 | trimSuffix "-") }} +{{- end }} + +{{- define "hugegraph.test.name" -}} +{{- printf "%s-test-connection" (include "hugegraph.fullname" . | trunc 47 | trimSuffix "-") }} +{{- end }} + +{{/* +PD Raft peers list: pod-0.svc.ns.svc:8610,... +Uses short headless DNS (cluster.local optional) resolvable inside the namespace. +*/}} +{{- define "hugegraph.pd.raftPeersList" -}} +{{- $peers := list -}} +{{- $replicas := int .Values.pd.replicas -}} +{{- $name := include "hugegraph.pd.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- $port := int .Values.pd.ports.raft -}} +{{- range $i := until $replicas -}} + {{- $peers = append $peers (printf "%s-%d.%s.%s.svc:%d" $name $i $name $ns $port) -}} +{{- end -}} +{{- join "," $peers -}} +{{- end }} + +{{/* +PD gRPC peers for Store/Server. +*/}} +{{- define "hugegraph.pd.grpcPeersList" -}} +{{- $peers := list -}} +{{- $replicas := int .Values.pd.replicas -}} +{{- $name := include "hugegraph.pd.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- $port := int .Values.pd.ports.grpc -}} +{{- range $i := until $replicas -}} + {{- $peers = append $peers (printf "%s-%d.%s.%s.svc:%d" $name $i $name $ns $port) -}} +{{- end -}} +{{- join "," $peers -}} +{{- end }} + +{{/* +PD REST endpoints for Server storage-readiness checks. +*/}} +{{- define "hugegraph.pd.restPeersList" -}} +{{- $peers := list -}} +{{- $replicas := int .Values.pd.replicas -}} +{{- $name := include "hugegraph.pd.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- $port := int .Values.pd.ports.rest -}} +{{- range $i := until $replicas -}} + {{- $peers = append $peers (printf "%s-%d.%s.%s.svc:%d" $name $i $name $ns $port) -}} +{{- end -}} +{{- join "," $peers -}} +{{- end }} + +{{/* +Initial store list for PD bootstrap: store-0.svc.ns.svc:8500,... +*/}} +{{- define "hugegraph.store.initialStoreList" -}} +{{- $peers := list -}} +{{- $replicas := int .Values.store.replicas -}} +{{- $name := include "hugegraph.store.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- $port := int .Values.store.ports.grpc -}} +{{- range $i := until $replicas -}} + {{- $peers = append $peers (printf "%s-%d.%s.%s.svc:%d" $name $i $name $ns $port) -}} +{{- end -}} +{{- join "," $peers -}} +{{- end }} + +{{/* +First store REST endpoint for STORE_REST / wait-partition. +*/}} +{{- define "hugegraph.store.restPrimary" -}} +{{- $name := include "hugegraph.store.name" . -}} +{{- $ns := .Release.Namespace -}} +{{- printf "%s-0.%s.%s.svc:%d" $name $name $ns (int .Values.store.ports.rest) -}} +{{- end }} + +{{/* +Quorum size: floor(replicas/2)+1 +*/}} +{{- define "hugegraph.pd.quorum" -}} +{{- add (div (int .Values.pd.replicas) 2) 1 -}} +{{- end }} + +{{/* +Render JAVA_OPTS only when explicitly configured. An empty value preserves the +image entrypoint's existing automatic JVM sizing behavior. +*/}} +{{- define "hugegraph.javaOptsEnv" -}} +{{- $javaOpts := default "" . -}} +{{- if ne (trim $javaOpts) "" -}} +- name: JAVA_OPTS + value: {{ $javaOpts | quote }} +{{- end -}} +{{- end }} + +{{/* +Keep the startup probe alive for the 300-second storage wait, the Server's +120-second start timeout, and 30 seconds of process overhead. Older stored +values remain accepted, but their rendered threshold is raised to this floor. +*/}} +{{- define "hugegraph.server.startupFailureThreshold" -}} +{{- $period := int .Values.server.probes.startup.periodSeconds -}} +{{- $configured := int .Values.server.probes.startup.failureThreshold -}} +{{- $minimum := div (add 449 $period) $period -}} +{{- max $configured $minimum -}} +{{- end }} + +{{/* +Optional probe tunables, emitted only when explicitly set. Kubernetes defaults +timeoutSeconds to 1 second, which a garbage-collection pause can exceed on a +loaded Server; operators need a supported way to raise it without forking the +chart. Only explicitly configured fields are rendered. +*/}} +{{- define "hugegraph.probeTuning" -}} +{{- if hasKey . "timeoutSeconds" }} +timeoutSeconds: {{ .timeoutSeconds }} +{{- end }} +{{- if hasKey . "initialDelaySeconds" }} +initialDelaySeconds: {{ .initialDelaySeconds }} +{{- end }} +{{- if hasKey . "successThreshold" }} +successThreshold: {{ .successThreshold }} +{{- end }} +{{- end }} + +{{/* +Resolve the ServiceAccount name for a component: an explicit name wins, +otherwise the generated one when create is true, otherwise "default". +*/}} +{{- define "hugegraph.serviceAccountName" -}} +{{- $sa := get .component "serviceAccount" | default dict -}} +{{- if get $sa "name" -}} +{{- get $sa "name" -}} +{{- else if (get $sa "create" | default false) -}} +{{- .name -}} +{{- else -}} +default +{{- end -}} +{{- end }} + +{{/* +The minimum Server replica count that a PDB must remain valid against. +*/}} +{{- define "hugegraph.server.replicaFloor" -}} +{{- if .Values.server.hpa.enabled -}} +{{- .Values.server.hpa.minReplicas -}} +{{- else -}} +{{- .Values.server.replicas -}} +{{- end -}} +{{- end }} + +{{/* +Cross-field validation that JSON Schema draft-07 cannot express. +*/}} +{{- define "hugegraph.validateValues" -}} +{{- $networkPolicy := get .Values "networkPolicy" | default dict -}} +{{- if (get $networkPolicy "enabled" | default false) -}} +{{- fail "networkPolicy.enabled=true is unsupported because this chart does not implement NetworkPolicy resources" -}} +{{- end -}} +{{- if and .Values.server.hpa.enabled (gt (int .Values.server.hpa.minReplicas) (int .Values.server.hpa.maxReplicas)) -}} +{{- fail "server.hpa.minReplicas must be less than or equal to server.hpa.maxReplicas" -}} +{{- end -}} +{{- if .Values.server.hpa.enabled -}} +{{- $serverResources := .Values.server.resources | default dict -}} +{{- $serverRequests := get $serverResources "requests" | default dict -}} +{{- if not (hasKey $serverRequests "cpu") -}} +{{- fail "server.resources.requests.cpu is required when server.hpa.enabled=true" -}} +{{- end -}} +{{- $cpuRequest := trim (toString (get $serverRequests "cpu")) -}} +{{- if or (eq $cpuRequest "") (hasPrefix "-" $cpuRequest) (regexMatch "^[+]?((0+([.]0*)?)|([.]0+))(([KMGTPE]i)|[numkMGTPE]|[eE][+-]?[0-9]+)?$" $cpuRequest) -}} +{{- fail "server.resources.requests.cpu must be strictly positive when server.hpa.enabled=true" -}} +{{- end -}} +{{- end -}} +{{/* +Only validate minAvailable where a PDB is actually rendered. The pd/store PDB +templates require replicas > 1, so a single-replica release never creates one +and must not be failed for a value that has no effect. +*/}} +{{- if and .Values.pd.pdb.enabled (gt (int .Values.pd.replicas) 1) (ge (int .Values.pd.pdb.minAvailable) (int .Values.pd.replicas)) -}} Review Comment: ‼️ This only rejects a PDB that blocks every disruption; it still accepts a budget below the PD Raft majority. For example, `pd.replicas=5` with `minAvailable=2` renders successfully and permits three voluntary evictions, leaving 2/5 members and no quorum. When the PD PDB is enabled, require `minAvailable >= floor(replicas / 2) + 1` (and define the supported even/odd replica contract), with boundary render tests for several replica counts. ########## helm/hugegraph/templates/store-statefulset.yaml: ########## @@ -0,0 +1,193 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "hugegraph.store.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: store +spec: + serviceName: {{ include "hugegraph.store.name" . }} + replicas: {{ .Values.store.replicas }} + podManagementPolicy: Parallel + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: store + template: + metadata: + labels: + {{- include "hugegraph.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: store + {{- with .Values.store.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} + {{- with .Values.store.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + automountServiceAccountToken: {{ get (get .Values.store "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} + serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.store "name" (include "hugegraph.store.name" .) ) }} + {{- with .Values.store.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + {{- with .Values.store.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.store.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.store.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.store.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.store.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.store.affinity }} + affinity: + {{- toYaml .Values.store.affinity | nindent 8 }} + {{- else }} + {{- with include "hugegraph.antiAffinity" (dict "mode" .Values.store.antiAffinity "component" "store" "labels" (include "hugegraph.selectorLabels" . | fromYaml)) }}{{ . | trim | nindent 6 }}{{- end }} + {{- end }} + initContainers: + - name: wait-for-pd + image: {{ .Values.store.waitImage | quote }} + {{- with .Values.store.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: + - sh + - -c + - | + set -eu + REQUIRED={{ include "hugegraph.pd.quorum" . }} + HEALTH_PEERS=$(echo "{{ include "hugegraph.pd.restPeersList" . }}" | tr ',' ' ') + TIMEOUT={{ .Values.store.waitTimeoutSeconds | default 900 }} + DEADLINE=$(( $(date +%s) + TIMEOUT )) + echo "Waiting for PD quorum (${REQUIRED}) among: ${HEALTH_PEERS}" + until [ "$( + ok=0 + for peer in ${HEALTH_PEERS}; do + if curl -fsS "http://${peer}/v1/health"; >/dev/null 2>&1; then Review Comment: ‼️ This counts responding PD processes, not a PD quorum: the current `/v1/health` handler returns success unconditionally and does not check for a Raft leader or member catch-up. Two REST endpoints can therefore release every Store while PD has no usable quorum; the PD/Store readiness probes use the same process-liveness signal, so a StatefulSet rolling update can also advance before the restarted member has rejoined safely. Please gate on an actual leader/quorum/catch-up-aware signal (for example, validated membership state) and add no-leader plus rolling-restart scenarios. ########## helm/hugegraph/templates/server-deployment.yaml: ########## @@ -0,0 +1,242 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- include "hugegraph.validateValues" . }} +{{- $restServer := .Values.server.restServer | default dict }} +{{- $minFreeMemory := "" }} +{{- $batchMaxWriteThreads := "" }} +{{- if hasKey $restServer "minFreeMemory" }} +{{- $minFreeMemory = toString (get $restServer "minFreeMemory") }} +{{- end }} +{{- if hasKey $restServer "batchMaxWriteThreads" }} +{{- $batchMaxWriteThreads = toString (get $restServer "batchMaxWriteThreads") }} +{{- end }} +{{- $customPort := ne (int .Values.server.port) 8080 }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "hugegraph.server.name" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: server +spec: + {{- if not .Values.server.hpa.enabled }} + replicas: {{ .Values.server.replicas }} + {{- end }} + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: server + template: + metadata: + labels: + {{- include "hugegraph.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: server + {{- with .Values.server.podLabels }}{{ toYaml . | nindent 8 }}{{- end }} + {{- with .Values.server.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + automountServiceAccountToken: {{ get (get .Values.server "serviceAccount" | default dict) "automountServiceAccountToken" | default false }} + serviceAccountName: {{ include "hugegraph.serviceAccountName" (dict "component" .Values.server "name" (include "hugegraph.server.name" .) ) }} + {{- with .Values.server.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + {{- with .Values.server.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.server.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.server.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.server.affinity }} + affinity: + {{- toYaml .Values.server.affinity | nindent 8 }} + {{- else }} + {{- with include "hugegraph.antiAffinity" (dict "mode" (.Values.server.antiAffinity | default "preferred") "component" "server" "labels" (include "hugegraph.selectorLabels" . | fromYaml)) }}{{ . | trim | nindent 6 }}{{- end }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: server + image: "{{ .Values.server.image.repository }}:{{ .Values.server.image.tag | default $.Chart.AppVersion }}" + imagePullPolicy: {{ .Values.server.image.pullPolicy }} + {{- with .Values.server.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if or .Values.server.auth.enabled + $customPort + (ne $minFreeMemory "") + (ne $batchMaxWriteThreads "") }} + command: + - /usr/bin/dumb-init + - -- + - /bin/bash + - -c + args: + - | + set -euo pipefail + {{- if .Values.server.auth.enabled }} + : "${PASSWORD:?auth Secret key 'password' must not be empty}" + {{- end }} + CONF=./conf/rest-server.properties + TMP=$(mktemp) + {{- if .Values.server.auth.enabled }} + FOUND_USE_PD=false + FOUND_PD_PEERS=false + {{- end }} + {{- if $customPort }} + FOUND_RESTSERVER_URL=false + {{- end }} + {{- if ne $minFreeMemory "" }} + FOUND_MIN_FREE_MEMORY=false + {{- end }} + {{- if ne $batchMaxWriteThreads "" }} + FOUND_BATCH_MAX_WRITE_THREADS=false + {{- end }} + while IFS= read -r LINE || [[ -n "${LINE}" ]]; do + case "${LINE}" in + {{- if .Values.server.auth.enabled }} + usePD=*) + printf 'usePD=true\n' >>"${TMP}" + FOUND_USE_PD=true + ;; + pd.peers=*) + printf 'pd.peers=%s\n' "${HG_SERVER_PD_PEERS}" >>"${TMP}" + FOUND_PD_PEERS=true + ;; + {{- end }} + {{- if $customPort }} + restserver.url=*) + printf 'restserver.url=http://0.0.0.0:%s\n' \ + {{ .Values.server.port | quote }} >>"${TMP}" + FOUND_RESTSERVER_URL=true + ;; + {{- end }} + {{- if ne $minFreeMemory "" }} + restserver.min_free_memory=*) + printf 'restserver.min_free_memory=%s\n' \ + {{ $minFreeMemory | quote }} >>"${TMP}" + FOUND_MIN_FREE_MEMORY=true + ;; + {{- end }} + {{- if ne $batchMaxWriteThreads "" }} + batch.max_write_threads=*) + printf 'batch.max_write_threads=%s\n' \ + {{ $batchMaxWriteThreads | quote }} >>"${TMP}" + FOUND_BATCH_MAX_WRITE_THREADS=true + ;; + {{- end }} + *) + printf '%s\n' "${LINE}" >>"${TMP}" + ;; + esac + done <"${CONF}" + {{- if .Values.server.auth.enabled }} + if [[ "${FOUND_USE_PD}" == false ]]; then + printf 'usePD=true\n' >>"${TMP}" + fi + if [[ "${FOUND_PD_PEERS}" == false ]]; then + printf 'pd.peers=%s\n' "${HG_SERVER_PD_PEERS}" >>"${TMP}" + fi + {{- end }} + {{- if $customPort }} + if [[ "${FOUND_RESTSERVER_URL}" == false ]]; then + printf 'restserver.url=http://0.0.0.0:%s\n' \ + {{ .Values.server.port | quote }} >>"${TMP}" + fi + {{- end }} + {{- if ne $minFreeMemory "" }} + if [[ "${FOUND_MIN_FREE_MEMORY}" == false ]]; then + printf 'restserver.min_free_memory=%s\n' \ + {{ $minFreeMemory | quote }} >>"${TMP}" + fi + {{- end }} + {{- if ne $batchMaxWriteThreads "" }} + if [[ "${FOUND_BATCH_MAX_WRITE_THREADS}" == false ]]; then + printf 'batch.max_write_threads=%s\n' \ + {{ $batchMaxWriteThreads | quote }} >>"${TMP}" + fi + {{- end }} + chmod 600 "${TMP}" + mv "${TMP}" "${CONF}" + exec ./docker-entrypoint.sh + {{- end }} + ports: + - name: http + containerPort: {{ .Values.server.port }} + env: + - name: HG_SERVER_BACKEND + value: {{ .Values.server.backend | quote }} + - name: HG_SERVER_PD_PEERS + value: {{ include "hugegraph.pd.grpcPeersList" . | quote }} + - name: HG_SERVER_PD_REST_ENDPOINT + value: {{ include "hugegraph.pd.restPeersList" . | quote }} + - name: STORE_REST + value: {{ include "hugegraph.store.restPrimary" . | quote }} + - name: HG_SERVER_INIT_STORE_ENABLED + value: {{ .Values.server.initStoreEnabled | quote }} + {{- with .Values.server.extraEnv }}{{ toYaml . | nindent 12 }}{{- end }} Review Comment: ⚠️ `extraEnv` is appended after chart-owned variables, while the schema allows duplicate names. A user can therefore add another `HG_SERVER_INIT_STORE_ENABLED=true` and bypass the chart's validated `false` contract, re-enabling concurrent init-store across Server replicas; the PD and Store templates expose the same override path for their identity/topology variables. Please reject chart-reserved names in each component's `extraEnv` (or render managed variables last with an explicit precedence contract) and add negative render tests. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
